IEMInternal.h@ 94791

Last change on this file since 94791 was 94768, checked in by vboxsync, 3 years ago
VMM/IEM: Split up IEMAll.cpp into a few more compilation units. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 145.7 KB

Line
1	/* $Id: IEMInternal.h 94768 2022-05-01 22:02:17Z vboxsync $ */
2	/** @file
3	* IEM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	#ifndef VMM_INCLUDED_SRC_include_IEMInternal_h
19	#define VMM_INCLUDED_SRC_include_IEMInternal_h
20	#ifndef RT_WITHOUT_PRAGMA_ONCE
21	# pragma once
22	#endif
23
24	#include <VBox/vmm/cpum.h>
25	#include <VBox/vmm/iem.h>
26	#include <VBox/vmm/pgm.h>
27	#include <VBox/vmm/stam.h>
28	#include <VBox/param.h>
29
30	#include <setjmp.h>
31
32
33	RT_C_DECLS_BEGIN
34
35
36	/** @defgroup grp_iem_int Internals
37	* @ingroup grp_iem
38	* @internal
39	* @{
40	*/
41
42	/** For expanding symbol in slickedit and other products tagging and
43	* crossreferencing IEM symbols. */
44	#ifndef IEM_STATIC
45	# define IEM_STATIC static
46	#endif
47
48	/** @def IEM_WITH_SETJMP
49	* Enables alternative status code handling using setjmps.
50	*
51	* This adds a bit of expense via the setjmp() call since it saves all the
52	* non-volatile registers. However, it eliminates return code checks and allows
53	* for more optimal return value passing (return regs instead of stack buffer).
54	*/
55	#if defined(DOXYGEN_RUNNING) \|\| defined(RT_OS_WINDOWS) \|\| 1
56	# define IEM_WITH_SETJMP
57	#endif
58
59	#define IEM_IMPLEMENTS_TASKSWITCH
60
61	/** @def IEM_WITH_3DNOW
62	* Includes the 3DNow decoding. */
63	#define IEM_WITH_3DNOW
64
65	/** @def IEM_WITH_THREE_0F_38
66	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
67	#define IEM_WITH_THREE_0F_38
68
69	/** @def IEM_WITH_THREE_0F_3A
70	* Includes the three byte opcode map for instrs starting with 0x0f 0x38. */
71	#define IEM_WITH_THREE_0F_3A
72
73	/** @def IEM_WITH_VEX
74	* Includes the VEX decoding. */
75	#define IEM_WITH_VEX
76
77	/** @def IEM_CFG_TARGET_CPU
78	* The minimum target CPU for the IEM emulation (IEMTARGETCPU_XXX value).
79	*
80	* By default we allow this to be configured by the user via the
81	* CPUM/GuestCpuName config string, but this comes at a slight cost during
82	* decoding. So, for applications of this code where there is no need to
83	* be dynamic wrt target CPU, just modify this define.
84	*/
85	#if !defined(IEM_CFG_TARGET_CPU) \|\| defined(DOXYGEN_RUNNING)
86	# define IEM_CFG_TARGET_CPU IEMTARGETCPU_DYNAMIC
87	#endif
88
89	//#define IEM_WITH_CODE_TLB // - work in progress
90	//#define IEM_WITH_DATA_TLB // - incomplete in progress
91
92
93	/** @def IEM_USE_UNALIGNED_DATA_ACCESS
94	* Use unaligned accesses instead of elaborate byte assembly. */
95	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86) \|\| defined(DOXYGEN_RUNNING)
96	# define IEM_USE_UNALIGNED_DATA_ACCESS
97	#endif
98
99	//#define IEM_LOG_MEMORY_WRITES
100
101	#if !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
102	/** Instruction statistics. */
103	typedef struct IEMINSTRSTATS
104	{
105	# define IEM_DO_INSTR_STAT(a_Name, a_szDesc) uint32_t a_Name;
106	# include "IEMInstructionStatisticsTmpl.h"
107	# undef IEM_DO_INSTR_STAT
108	} IEMINSTRSTATS;
109	#else
110	struct IEMINSTRSTATS;
111	typedef struct IEMINSTRSTATS IEMINSTRSTATS;
112	#endif
113	/** Pointer to IEM instruction statistics. */
114	typedef IEMINSTRSTATS *PIEMINSTRSTATS;
115
116
117	/** @name IEMTARGETCPU_EFL_BEHAVIOR_XXX - IEMCPU::idxTargetCpuEflFlavour
118	* @{ */
119	#define IEMTARGETCPU_EFL_BEHAVIOR_NATIVE 0 /*< Native x86 EFLAGS result; Intel EFLAGS when on non-x86 hosts. /
120	#define IEMTARGETCPU_EFL_BEHAVIOR_INTEL 1 /*< Intel EFLAGS result. /
121	#define IEMTARGETCPU_EFL_BEHAVIOR_AMD 2 /*< AMD EFLAGS result /
122	#define IEMTARGETCPU_EFL_BEHAVIOR_RESERVED 3 /*< Reserved/dummy entry slot that's the same as 0. /
123	#define IEMTARGETCPU_EFL_BEHAVIOR_MASK 3 /*< For masking the index before use. /
124	/** Selects the right variant from a_aArray.
125	* pVCpu is implicit in the caller context. */
126	#define IEMTARGETCPU_EFL_BEHAVIOR_SELECT(a_aArray) \
127	(a_aArray[pVCpu->iem.s.idxTargetCpuEflFlavour & IEMTARGETCPU_EFL_BEHAVIOR_MASK])
128	/** @} */
129
130
131	/**
132	* Extended operand mode that includes a representation of 8-bit.
133	*
134	* This is used for packing down modes when invoking some C instruction
135	* implementations.
136	*/
137	typedef enum IEMMODEX
138	{
139	IEMMODEX_16BIT = IEMMODE_16BIT,
140	IEMMODEX_32BIT = IEMMODE_32BIT,
141	IEMMODEX_64BIT = IEMMODE_64BIT,
142	IEMMODEX_8BIT
143	} IEMMODEX;
144	AssertCompileSize(IEMMODEX, 4);
145
146
147	/**
148	* Branch types.
149	*/
150	typedef enum IEMBRANCH
151	{
152	IEMBRANCH_JUMP = 1,
153	IEMBRANCH_CALL,
154	IEMBRANCH_TRAP,
155	IEMBRANCH_SOFTWARE_INT,
156	IEMBRANCH_HARDWARE_INT
157	} IEMBRANCH;
158	AssertCompileSize(IEMBRANCH, 4);
159
160
161	/**
162	* INT instruction types.
163	*/
164	typedef enum IEMINT
165	{
166	/** INT n instruction (opcode 0xcd imm). */
167	IEMINT_INTN = 0,
168	/** Single byte INT3 instruction (opcode 0xcc). */
169	IEMINT_INT3 = IEM_XCPT_FLAGS_BP_INSTR,
170	/** Single byte INTO instruction (opcode 0xce). */
171	IEMINT_INTO = IEM_XCPT_FLAGS_OF_INSTR,
172	/** Single byte INT1 (ICEBP) instruction (opcode 0xf1). */
173	IEMINT_INT1 = IEM_XCPT_FLAGS_ICEBP_INSTR
174	} IEMINT;
175	AssertCompileSize(IEMINT, 4);
176
177
178	/**
179	* A FPU result.
180	*/
181	typedef struct IEMFPURESULT
182	{
183	/** The output value. */
184	RTFLOAT80U r80Result;
185	/** The output status. */
186	uint16_t FSW;
187	} IEMFPURESULT;
188	AssertCompileMemberOffset(IEMFPURESULT, FSW, 10);
189	/** Pointer to a FPU result. */
190	typedef IEMFPURESULT *PIEMFPURESULT;
191	/** Pointer to a const FPU result. */
192	typedef IEMFPURESULT const *PCIEMFPURESULT;
193
194
195	/**
196	* A FPU result consisting of two output values and FSW.
197	*/
198	typedef struct IEMFPURESULTTWO
199	{
200	/** The first output value. */
201	RTFLOAT80U r80Result1;
202	/** The output status. */
203	uint16_t FSW;
204	/** The second output value. */
205	RTFLOAT80U r80Result2;
206	} IEMFPURESULTTWO;
207	AssertCompileMemberOffset(IEMFPURESULTTWO, FSW, 10);
208	AssertCompileMemberOffset(IEMFPURESULTTWO, r80Result2, 12);
209	/** Pointer to a FPU result consisting of two output values and FSW. */
210	typedef IEMFPURESULTTWO *PIEMFPURESULTTWO;
211	/** Pointer to a const FPU result consisting of two output values and FSW. */
212	typedef IEMFPURESULTTWO const *PCIEMFPURESULTTWO;
213
214
215	/**
216	* IEM TLB entry.
217	*
218	* Lookup assembly:
219	* @code{.asm}
220	; Calculate tag.
221	mov rax, [VA]
222	shl rax, 16
223	shr rax, 16 + X86_PAGE_SHIFT
224	or rax, [uTlbRevision]
225
226	; Do indexing.
227	movzx ecx, al
228	lea rcx, [pTlbEntries + rcx]
229
230	; Check tag.
231	cmp [rcx + IEMTLBENTRY.uTag], rax
232	jne .TlbMiss
233
234	; Check access.
235	mov rax, ACCESS_FLAGS \| MAPPING_R3_NOT_VALID \| 0xffffff00
236	and rax, [rcx + IEMTLBENTRY.fFlagsAndPhysRev]
237	cmp rax, [uTlbPhysRev]
238	jne .TlbMiss
239
240	; Calc address and we're done.
241	mov eax, X86_PAGE_OFFSET_MASK
242	and eax, [VA]
243	or rax, [rcx + IEMTLBENTRY.pMappingR3]
244	%ifdef VBOX_WITH_STATISTICS
245	inc qword [cTlbHits]
246	%endif
247	jmp .Done
248
249	.TlbMiss:
250	mov r8d, ACCESS_FLAGS
251	mov rdx, [VA]
252	mov rcx, [pVCpu]
253	call iemTlbTypeMiss
254	.Done:
255
256	@endcode
257	*
258	*/
259	typedef struct IEMTLBENTRY
260	{
261	/** The TLB entry tag.
262	* Bits 35 thru 0 are made up of the virtual address shifted right 12 bits, this
263	* is ASSUMING a virtual address width of 48 bits.
264	*
265	* Bits 63 thru 36 are made up of the TLB revision (zero means invalid).
266	*
267	* The TLB lookup code uses the current TLB revision, which won't ever be zero,
268	* enabling an extremely cheap TLB invalidation most of the time. When the TLB
269	* revision wraps around though, the tags needs to be zeroed.
270	*
271	* @note Try use SHRD instruction? After seeing
272	* https://gmplib.org/~tege/x86-timing.pdf, maybe not.
273	*
274	* @todo This will need to be reorganized for 57-bit wide virtual address and
275	* PCID (currently 12 bits) and ASID (currently 6 bits) support. We'll
276	* have to move the TLB entry versioning entirely to the
277	* fFlagsAndPhysRev member then, 57 bit wide VAs means we'll only have
278	* 19 bits left (64 - 57 + 12 = 19) and they'll almost entire be
279	* consumed by PCID and ASID (12 + 6 = 18).
280	*/
281	uint64_t uTag;
282	/** Access flags and physical TLB revision.
283	*
284	* - Bit 0 - page tables - not executable (X86_PTE_PAE_NX).
285	* - Bit 1 - page tables - not writable (complemented X86_PTE_RW).
286	* - Bit 2 - page tables - not user (complemented X86_PTE_US).
287	* - Bit 3 - pgm phys/virt - not directly writable.
288	* - Bit 4 - pgm phys page - not directly readable.
289	* - Bit 5 - currently unused.
290	* - Bit 6 - page tables - not dirty (complemented X86_PTE_D).
291	* - Bit 7 - tlb entry - pMappingR3 member not valid.
292	* - Bits 63 thru 8 are used for the physical TLB revision number.
293	*
294	* We're using complemented bit meanings here because it makes it easy to check
295	* whether special action is required. For instance a user mode write access
296	* would do a "TEST fFlags, (X86_PTE_RW \| X86_PTE_US \| X86_PTE_D)" and a
297	* non-zero result would mean special handling needed because either it wasn't
298	* writable, or it wasn't user, or the page wasn't dirty. A user mode read
299	* access would do "TEST fFlags, X86_PTE_US"; and a kernel mode read wouldn't
300	* need to check any PTE flag.
301	*/
302	uint64_t fFlagsAndPhysRev;
303	/** The guest physical page address. */
304	uint64_t GCPhys;
305	/** Pointer to the ring-3 mapping (possibly also valid in ring-0). */
306	R3PTRTYPE(uint8_t *) pbMappingR3;
307	#if HC_ARCH_BITS == 32
308	uint32_t u32Padding1;
309	#endif
310	} IEMTLBENTRY;
311	AssertCompileSize(IEMTLBENTRY, 32);
312	/** Pointer to an IEM TLB entry. */
313	typedef IEMTLBENTRY *PIEMTLBENTRY;
314
315	/** @name IEMTLBE_F_XXX - TLB entry flags (IEMTLBENTRY::fFlagsAndPhysRev)
316	* @{ */
317	#define IEMTLBE_F_PT_NO_EXEC RT_BIT_64(0) /*< Page tables: Not executable. /
318	#define IEMTLBE_F_PT_NO_WRITE RT_BIT_64(1) /*< Page tables: Not writable. /
319	#define IEMTLBE_F_PT_NO_USER RT_BIT_64(2) /*< Page tables: Not user accessible (supervisor only). /
320	#define IEMTLBE_F_PG_NO_WRITE RT_BIT_64(3) /*< Phys page: Not writable (access handler, ROM, whatever). /
321	#define IEMTLBE_F_PG_NO_READ RT_BIT_64(4) /*< Phys page: Not readable (MMIO / access handler, ROM) /
322	#define IEMTLBE_F_PT_NO_DIRTY RT_BIT_64(6) /*< Page tables: Not dirty (needs to be made dirty on write). /
323	#define IEMTLBE_F_NO_MAPPINGR3 RT_BIT_64(7) /*< TLB entry: The IEMTLBENTRY::pMappingR3 member is invalid. /
324	#define IEMTLBE_F_PHYS_REV UINT64_C(0xffffffffffffff00) /*< Physical revision mask. /
325	/** @} */
326
327
328	/**
329	* An IEM TLB.
330	*
331	* We've got two of these, one for data and one for instructions.
332	*/
333	typedef struct IEMTLB
334	{
335	/** The TLB entries.
336	* We've choosen 256 because that way we can obtain the result directly from a
337	* 8-bit register without an additional AND instruction. */
338	IEMTLBENTRY aEntries[256];
339	/** The TLB revision.
340	* This is actually only 28 bits wide (see IEMTLBENTRY::uTag) and is incremented
341	* by adding RT_BIT_64(36) to it. When it wraps around and becomes zero, all
342	* the tags in the TLB must be zeroed and the revision set to RT_BIT_64(36).
343	* (The revision zero indicates an invalid TLB entry.)
344	*
345	* The initial value is choosen to cause an early wraparound. */
346	uint64_t uTlbRevision;
347	/** The TLB physical address revision - shadow of PGM variable.
348	*
349	* This is actually only 56 bits wide (see IEMTLBENTRY::fFlagsAndPhysRev) and is
350	* incremented by adding RT_BIT_64(8). When it wraps around and becomes zero,
351	* a rendezvous is called and each CPU wipe the IEMTLBENTRY::pMappingR3 as well
352	* as IEMTLBENTRY::fFlagsAndPhysRev bits 63 thru 8, 4, and 3.
353	*
354	* The initial value is choosen to cause an early wraparound. */
355	uint64_t volatile uTlbPhysRev;
356
357	/* Statistics: */
358
359	/** TLB hits (VBOX_WITH_STATISTICS only). */
360	uint64_t cTlbHits;
361	/** TLB misses. */
362	uint32_t cTlbMisses;
363	/** Slow read path. */
364	uint32_t cTlbSlowReadPath;
365	#if 0
366	/** TLB misses because of tag mismatch. */
367	uint32_t cTlbMissesTag;
368	/** TLB misses because of virtual access violation. */
369	uint32_t cTlbMissesVirtAccess;
370	/** TLB misses because of dirty bit. */
371	uint32_t cTlbMissesDirty;
372	/** TLB misses because of MMIO */
373	uint32_t cTlbMissesMmio;
374	/** TLB misses because of write access handlers. */
375	uint32_t cTlbMissesWriteHandler;
376	/** TLB misses because no r3(/r0) mapping. */
377	uint32_t cTlbMissesMapping;
378	#endif
379	/** Alignment padding. */
380	uint32_t au32Padding[3+5];
381	} IEMTLB;
382	AssertCompileSizeAlignment(IEMTLB, 64);
383	/** IEMTLB::uTlbRevision increment. */
384	#define IEMTLB_REVISION_INCR RT_BIT_64(36)
385	/** IEMTLB::uTlbPhysRev increment. */
386	#define IEMTLB_PHYS_REV_INCR RT_BIT_64(8)
387
388
389	/**
390	* The per-CPU IEM state.
391	*/
392	typedef struct IEMCPU
393	{
394	/** Info status code that needs to be propagated to the IEM caller.
395	* This cannot be passed internally, as it would complicate all success
396	* checks within the interpreter making the code larger and almost impossible
397	* to get right. Instead, we'll store status codes to pass on here. Each
398	* source of these codes will perform appropriate sanity checks. */
399	int32_t rcPassUp; /* 0x00 */
400
401	/** The current CPU execution mode (CS). */
402	IEMMODE enmCpuMode; /* 0x04 */
403	/** The CPL. */
404	uint8_t uCpl; /* 0x05 */
405
406	/** Whether to bypass access handlers or not. */
407	bool fBypassHandlers; /* 0x06 */
408	/** Whether to disregard the lock prefix (implied or not). */
409	bool fDisregardLock; /* 0x07 */
410
411	/** @name Decoder state.
412	* @{ */
413	#ifdef IEM_WITH_CODE_TLB
414	/** The offset of the next instruction byte. */
415	uint32_t offInstrNextByte; /* 0x08 */
416	/** The number of bytes available at pbInstrBuf for the current instruction.
417	* This takes the max opcode length into account so that doesn't need to be
418	* checked separately. */
419	uint32_t cbInstrBuf; /* 0x0c */
420	/** Pointer to the page containing RIP, user specified buffer or abOpcode.
421	* This can be NULL if the page isn't mappable for some reason, in which
422	* case we'll do fallback stuff.
423	*
424	* If we're executing an instruction from a user specified buffer,
425	* IEMExecOneWithPrefetchedByPC and friends, this is not necessarily a page
426	* aligned pointer but pointer to the user data.
427	*
428	* For instructions crossing pages, this will start on the first page and be
429	* advanced to the next page by the time we've decoded the instruction. This
430	* therefore precludes stuff like <tt>pbInstrBuf[offInstrNextByte + cbInstrBuf - cbCurInstr]</tt>
431	*/
432	uint8_t const pbInstrBuf; / 0x10 */
433	# if ARCH_BITS == 32
434	uint32_t uInstrBufHigh; /** The high dword of the host context pbInstrBuf member. */
435	# endif
436	/** The program counter corresponding to pbInstrBuf.
437	* This is set to a non-canonical address when we need to invalidate it. */
438	uint64_t uInstrBufPc; /* 0x18 */
439	/** The number of bytes available at pbInstrBuf in total (for IEMExecLots).
440	* This takes the CS segment limit into account. */
441	uint16_t cbInstrBufTotal; /* 0x20 */
442	/** Offset into pbInstrBuf of the first byte of the current instruction.
443	* Can be negative to efficiently handle cross page instructions. */
444	int16_t offCurInstrStart; /* 0x22 */
445
446	/** The prefix mask (IEM_OP_PRF_XXX). */
447	uint32_t fPrefixes; /* 0x24 */
448	/** The extra REX ModR/M register field bit (REX.R << 3). */
449	uint8_t uRexReg; /* 0x28 */
450	/** The extra REX ModR/M r/m field, SIB base and opcode reg bit
451	* (REX.B << 3). */
452	uint8_t uRexB; /* 0x29 */
453	/** The extra REX SIB index field bit (REX.X << 3). */
454	uint8_t uRexIndex; /* 0x2a */
455
456	/** The effective segment register (X86_SREG_XXX). */
457	uint8_t iEffSeg; /* 0x2b */
458
459	/** The offset of the ModR/M byte relative to the start of the instruction. */
460	uint8_t offModRm; /* 0x2c */
461	#else
462	/** The size of what has currently been fetched into abOpcode. */
463	uint8_t cbOpcode; /* 0x08 */
464	/** The current offset into abOpcode. */
465	uint8_t offOpcode; /* 0x09 */
466	/** The offset of the ModR/M byte relative to the start of the instruction. */
467	uint8_t offModRm; /* 0x0a */
468
469	/** The effective segment register (X86_SREG_XXX). */
470	uint8_t iEffSeg; /* 0x0b */
471
472	/** The prefix mask (IEM_OP_PRF_XXX). */
473	uint32_t fPrefixes; /* 0x0c */
474	/** The extra REX ModR/M register field bit (REX.R << 3). */
475	uint8_t uRexReg; /* 0x10 */
476	/** The extra REX ModR/M r/m field, SIB base and opcode reg bit
477	* (REX.B << 3). */
478	uint8_t uRexB; /* 0x11 */
479	/** The extra REX SIB index field bit (REX.X << 3). */
480	uint8_t uRexIndex; /* 0x12 */
481
482	#endif
483
484	/** The effective operand mode. */
485	IEMMODE enmEffOpSize; /* 0x2d, 0x13 */
486	/** The default addressing mode. */
487	IEMMODE enmDefAddrMode; /* 0x2e, 0x14 */
488	/** The effective addressing mode. */
489	IEMMODE enmEffAddrMode; /* 0x2f, 0x15 */
490	/** The default operand mode. */
491	IEMMODE enmDefOpSize; /* 0x30, 0x16 */
492
493	/** Prefix index (VEX.pp) for two byte and three byte tables. */
494	uint8_t idxPrefix; /* 0x31, 0x17 */
495	/** 3rd VEX/EVEX/XOP register.
496	* Please use IEM_GET_EFFECTIVE_VVVV to access. */
497	uint8_t uVex3rdReg; /* 0x32, 0x18 */
498	/** The VEX/EVEX/XOP length field. */
499	uint8_t uVexLength; /* 0x33, 0x19 */
500	/** Additional EVEX stuff. */
501	uint8_t fEvexStuff; /* 0x34, 0x1a */
502
503	/** Explicit alignment padding. */
504	uint8_t abAlignment2a[1]; /* 0x35, 0x1b */
505	/** The FPU opcode (FOP). */
506	uint16_t uFpuOpcode; /* 0x36, 0x1c */
507	#ifndef IEM_WITH_CODE_TLB
508	/** Explicit alignment padding. */
509	uint8_t abAlignment2b[2]; /* 0x1e */
510	#endif
511
512	/** The opcode bytes. */
513	uint8_t abOpcode[15]; /* 0x48, 0x20 */
514	/** Explicit alignment padding. */
515	#ifdef IEM_WITH_CODE_TLB
516	uint8_t abAlignment2c[0x48 - 0x47]; /* 0x37 */
517	#else
518	uint8_t abAlignment2c[0x48 - 0x2f]; /* 0x2f */
519	#endif
520	/** @} */
521
522
523	/** The flags of the current exception / interrupt. */
524	uint32_t fCurXcpt; /* 0x48, 0x48 */
525	/** The current exception / interrupt. */
526	uint8_t uCurXcpt;
527	/** Exception / interrupt recursion depth. */
528	int8_t cXcptRecursions;
529
530	/** The number of active guest memory mappings. */
531	uint8_t cActiveMappings;
532	/** The next unused mapping index. */
533	uint8_t iNextMapping;
534	/** Records for tracking guest memory mappings. */
535	struct
536	{
537	/** The address of the mapped bytes. */
538	void *pv;
539	/** The access flags (IEM_ACCESS_XXX).
540	* IEM_ACCESS_INVALID if the entry is unused. */
541	uint32_t fAccess;
542	#if HC_ARCH_BITS == 64
543	uint32_t u32Alignment4; /*< Alignment padding. /
544	#endif
545	} aMemMappings[3];
546
547	/** Locking records for the mapped memory. */
548	union
549	{
550	PGMPAGEMAPLOCK Lock;
551	uint64_t au64Padding[2];
552	} aMemMappingLocks[3];
553
554	/** Bounce buffer info.
555	* This runs in parallel to aMemMappings. */
556	struct
557	{
558	/** The physical address of the first byte. */
559	RTGCPHYS GCPhysFirst;
560	/** The physical address of the second page. */
561	RTGCPHYS GCPhysSecond;
562	/** The number of bytes in the first page. */
563	uint16_t cbFirst;
564	/** The number of bytes in the second page. */
565	uint16_t cbSecond;
566	/** Whether it's unassigned memory. */
567	bool fUnassigned;
568	/** Explicit alignment padding. */
569	bool afAlignment5[3];
570	} aMemBbMappings[3];
571
572	/** Bounce buffer storage.
573	* This runs in parallel to aMemMappings and aMemBbMappings. */
574	struct
575	{
576	uint8_t ab[512];
577	} aBounceBuffers[3];
578
579
580	/** Pointer set jump buffer - ring-3 context. */
581	R3PTRTYPE(jmp_buf *) pJmpBufR3;
582	/** Pointer set jump buffer - ring-0 context. */
583	R0PTRTYPE(jmp_buf *) pJmpBufR0;
584
585	/** @todo Should move this near @a fCurXcpt later. */
586	/** The CR2 for the current exception / interrupt. */
587	uint64_t uCurXcptCr2;
588	/** The error code for the current exception / interrupt. */
589	uint32_t uCurXcptErr;
590
591	/** @name Statistics
592	* @{ */
593	/** The number of instructions we've executed. */
594	uint32_t cInstructions;
595	/** The number of potential exits. */
596	uint32_t cPotentialExits;
597	/** The number of bytes data or stack written (mostly for IEMExecOneEx).
598	* This may contain uncommitted writes. */
599	uint32_t cbWritten;
600	/** Counts the VERR_IEM_INSTR_NOT_IMPLEMENTED returns. */
601	uint32_t cRetInstrNotImplemented;
602	/** Counts the VERR_IEM_ASPECT_NOT_IMPLEMENTED returns. */
603	uint32_t cRetAspectNotImplemented;
604	/** Counts informational statuses returned (other than VINF_SUCCESS). */
605	uint32_t cRetInfStatuses;
606	/** Counts other error statuses returned. */
607	uint32_t cRetErrStatuses;
608	/** Number of times rcPassUp has been used. */
609	uint32_t cRetPassUpStatus;
610	/** Number of times RZ left with instruction commit pending for ring-3. */
611	uint32_t cPendingCommit;
612	/** Number of long jumps. */
613	uint32_t cLongJumps;
614	/** @} */
615
616	/** @name Target CPU information.
617	* @{ */
618	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
619	/** The target CPU. */
620	uint8_t uTargetCpu;
621	#else
622	uint8_t bTargetCpuPadding;
623	#endif
624	/** For selecting assembly works matching the target CPU EFLAGS behaviour, see
625	* IEMTARGETCPU_EFL_BEHAVIOR_XXX for values. This is for instance used for the
626	* BSF & BSR instructions where AMD and Intel CPUs produce different EFLAGS. */
627	uint8_t idxTargetCpuEflFlavour;
628
629	/** The CPU vendor. */
630	CPUMCPUVENDOR enmCpuVendor;
631	/** @} */
632
633	/** @name Host CPU information.
634	* @{ */
635	/** The CPU vendor. */
636	CPUMCPUVENDOR enmHostCpuVendor;
637	/** @} */
638
639	/** Counts RDMSR \#GP(0) LogRel(). */
640	uint8_t cLogRelRdMsr;
641	/** Counts WRMSR \#GP(0) LogRel(). */
642	uint8_t cLogRelWrMsr;
643	/** Alignment padding. */
644	uint8_t abAlignment8[50];
645
646	/** Data TLB.
647	* @remarks Must be 64-byte aligned. */
648	IEMTLB DataTlb;
649	/** Instruction TLB.
650	* @remarks Must be 64-byte aligned. */
651	IEMTLB CodeTlb;
652
653	#if defined(VBOX_WITH_STATISTICS) && !defined(IN_TSTVMSTRUCT) && !defined(DOXYGEN_RUNNING)
654	/** Instruction statistics for ring-0/raw-mode. */
655	IEMINSTRSTATS StatsRZ;
656	/** Instruction statistics for ring-3. */
657	IEMINSTRSTATS StatsR3;
658	#endif
659	} IEMCPU;
660	AssertCompileMemberOffset(IEMCPU, fCurXcpt, 0x48);
661	AssertCompileMemberAlignment(IEMCPU, DataTlb, 64);
662	AssertCompileMemberAlignment(IEMCPU, CodeTlb, 64);
663	/** Pointer to the per-CPU IEM state. */
664	typedef IEMCPU *PIEMCPU;
665	/** Pointer to the const per-CPU IEM state. */
666	typedef IEMCPU const *PCIEMCPU;
667
668
669	/** @def IEM_GET_CTX
670	* Gets the guest CPU context for the calling EMT.
671	* @returns PCPUMCTX
672	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
673	*/
674	#define IEM_GET_CTX(a_pVCpu) (&(a_pVCpu)->cpum.GstCtx)
675
676	/** @def IEM_CTX_ASSERT
677	* Asserts that the @a a_fExtrnMbz is present in the CPU context.
678	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
679	* @param a_fExtrnMbz The mask of CPUMCTX_EXTRN_XXX flags that must be zero.
680	*/
681	#define IEM_CTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
682	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", (a_pVCpu)->cpum.GstCtx.fExtrn, \
683	(a_fExtrnMbz)))
684
685	/** @def IEM_CTX_IMPORT_RET
686	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
687	*
688	* Will call the keep to import the bits as needed.
689	*
690	* Returns on import failure.
691	*
692	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
693	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
694	*/
695	#define IEM_CTX_IMPORT_RET(a_pVCpu, a_fExtrnImport) \
696	do { \
697	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
698	{ /* likely */ } \
699	else \
700	{ \
701	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
702	AssertRCReturn(rcCtxImport, rcCtxImport); \
703	} \
704	} while (0)
705
706	/** @def IEM_CTX_IMPORT_NORET
707	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
708	*
709	* Will call the keep to import the bits as needed.
710	*
711	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
712	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
713	*/
714	#define IEM_CTX_IMPORT_NORET(a_pVCpu, a_fExtrnImport) \
715	do { \
716	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
717	{ /* likely */ } \
718	else \
719	{ \
720	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
721	AssertLogRelRC(rcCtxImport); \
722	} \
723	} while (0)
724
725	/** @def IEM_CTX_IMPORT_JMP
726	* Makes sure the CPU context bits given by @a a_fExtrnImport are imported.
727	*
728	* Will call the keep to import the bits as needed.
729	*
730	* Jumps on import failure.
731	*
732	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
733	* @param a_fExtrnImport The mask of CPUMCTX_EXTRN_XXX flags to import.
734	*/
735	#define IEM_CTX_IMPORT_JMP(a_pVCpu, a_fExtrnImport) \
736	do { \
737	if (!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnImport))) \
738	{ /* likely */ } \
739	else \
740	{ \
741	int rcCtxImport = CPUMImportGuestStateOnDemand(a_pVCpu, a_fExtrnImport); \
742	AssertRCStmt(rcCtxImport, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), rcCtxImport)); \
743	} \
744	} while (0)
745
746
747
748	/** @def IEM_GET_TARGET_CPU
749	* Gets the current IEMTARGETCPU value.
750	* @returns IEMTARGETCPU value.
751	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
752	*/
753	#if IEM_CFG_TARGET_CPU != IEMTARGETCPU_DYNAMIC
754	# define IEM_GET_TARGET_CPU(a_pVCpu) (IEM_CFG_TARGET_CPU)
755	#else
756	# define IEM_GET_TARGET_CPU(a_pVCpu) ((a_pVCpu)->iem.s.uTargetCpu)
757	#endif
758
759	/** @def IEM_GET_INSTR_LEN
760	* Gets the instruction length. */
761	#ifdef IEM_WITH_CODE_TLB
762	# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offInstrNextByte - (uint32_t)(int32_t)(a_pVCpu)->iem.s.offCurInstrStart)
763	#else
764	# define IEM_GET_INSTR_LEN(a_pVCpu) ((a_pVCpu)->iem.s.offOpcode)
765	#endif
766
767
768	/**
769	* Shared per-VM IEM data.
770	*/
771	typedef struct IEM
772	{
773	/** The VMX APIC-access page handler type. */
774	PGMPHYSHANDLERTYPE hVmxApicAccessPage;
775	} IEM;
776
777
778
779	/** @name IEM_ACCESS_XXX - Access details.
780	* @{ */
781	#define IEM_ACCESS_INVALID UINT32_C(0x000000ff)
782	#define IEM_ACCESS_TYPE_READ UINT32_C(0x00000001)
783	#define IEM_ACCESS_TYPE_WRITE UINT32_C(0x00000002)
784	#define IEM_ACCESS_TYPE_EXEC UINT32_C(0x00000004)
785	#define IEM_ACCESS_TYPE_MASK UINT32_C(0x00000007)
786	#define IEM_ACCESS_WHAT_CODE UINT32_C(0x00000010)
787	#define IEM_ACCESS_WHAT_DATA UINT32_C(0x00000020)
788	#define IEM_ACCESS_WHAT_STACK UINT32_C(0x00000030)
789	#define IEM_ACCESS_WHAT_SYS UINT32_C(0x00000040)
790	#define IEM_ACCESS_WHAT_MASK UINT32_C(0x00000070)
791	/** The writes are partial, so if initialize the bounce buffer with the
792	* orignal RAM content. */
793	#define IEM_ACCESS_PARTIAL_WRITE UINT32_C(0x00000100)
794	/** Used in aMemMappings to indicate that the entry is bounce buffered. */
795	#define IEM_ACCESS_BOUNCE_BUFFERED UINT32_C(0x00000200)
796	/** Bounce buffer with ring-3 write pending, first page. */
797	#define IEM_ACCESS_PENDING_R3_WRITE_1ST UINT32_C(0x00000400)
798	/** Bounce buffer with ring-3 write pending, second page. */
799	#define IEM_ACCESS_PENDING_R3_WRITE_2ND UINT32_C(0x00000800)
800	/** Valid bit mask. */
801	#define IEM_ACCESS_VALID_MASK UINT32_C(0x00000fff)
802	/** Read+write data alias. */
803	#define IEM_ACCESS_DATA_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
804	/** Write data alias. */
805	#define IEM_ACCESS_DATA_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_DATA)
806	/** Read data alias. */
807	#define IEM_ACCESS_DATA_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_DATA)
808	/** Instruction fetch alias. */
809	#define IEM_ACCESS_INSTRUCTION (IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_WHAT_CODE)
810	/** Stack write alias. */
811	#define IEM_ACCESS_STACK_W (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
812	/** Stack read alias. */
813	#define IEM_ACCESS_STACK_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_STACK)
814	/** Stack read+write alias. */
815	#define IEM_ACCESS_STACK_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_STACK)
816	/** Read system table alias. */
817	#define IEM_ACCESS_SYS_R (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_WHAT_SYS)
818	/** Read+write system table alias. */
819	#define IEM_ACCESS_SYS_RW (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_WHAT_SYS)
820	/** @} */
821
822	/** @name Prefix constants (IEMCPU::fPrefixes)
823	* @{ */
824	#define IEM_OP_PRF_SEG_CS RT_BIT_32(0) /*< CS segment prefix (0x2e). /
825	#define IEM_OP_PRF_SEG_SS RT_BIT_32(1) /*< SS segment prefix (0x36). /
826	#define IEM_OP_PRF_SEG_DS RT_BIT_32(2) /*< DS segment prefix (0x3e). /
827	#define IEM_OP_PRF_SEG_ES RT_BIT_32(3) /*< ES segment prefix (0x26). /
828	#define IEM_OP_PRF_SEG_FS RT_BIT_32(4) /*< FS segment prefix (0x64). /
829	#define IEM_OP_PRF_SEG_GS RT_BIT_32(5) /*< GS segment prefix (0x65). /
830	#define IEM_OP_PRF_SEG_MASK UINT32_C(0x3f)
831
832	#define IEM_OP_PRF_SIZE_OP RT_BIT_32(8) /*< Operand size prefix (0x66). /
833	#define IEM_OP_PRF_SIZE_REX_W RT_BIT_32(9) /*< REX.W prefix (0x48-0x4f). /
834	#define IEM_OP_PRF_SIZE_ADDR RT_BIT_32(10) /*< Address size prefix (0x67). /
835
836	#define IEM_OP_PRF_LOCK RT_BIT_32(16) /*< Lock prefix (0xf0). /
837	#define IEM_OP_PRF_REPNZ RT_BIT_32(17) /*< Repeat-not-zero prefix (0xf2). /
838	#define IEM_OP_PRF_REPZ RT_BIT_32(18) /*< Repeat-if-zero prefix (0xf3). /
839
840	#define IEM_OP_PRF_REX RT_BIT_32(24) /*< Any REX prefix (0x40-0x4f). /
841	#define IEM_OP_PRF_REX_R RT_BIT_32(25) /*< REX.R prefix (0x44,0x45,0x46,0x47,0x4c,0x4d,0x4e,0x4f). /
842	#define IEM_OP_PRF_REX_B RT_BIT_32(26) /*< REX.B prefix (0x41,0x43,0x45,0x47,0x49,0x4b,0x4d,0x4f). /
843	#define IEM_OP_PRF_REX_X RT_BIT_32(27) /*< REX.X prefix (0x42,0x43,0x46,0x47,0x4a,0x4b,0x4e,0x4f). /
844	/** Mask with all the REX prefix flags.
845	* This is generally for use when needing to undo the REX prefixes when they
846	* are followed legacy prefixes and therefore does not immediately preceed
847	* the first opcode byte.
848	* For testing whether any REX prefix is present, use IEM_OP_PRF_REX instead. */
849	#define IEM_OP_PRF_REX_MASK (IEM_OP_PRF_REX \| IEM_OP_PRF_REX_R \| IEM_OP_PRF_REX_B \| IEM_OP_PRF_REX_X \| IEM_OP_PRF_SIZE_REX_W )
850
851	#define IEM_OP_PRF_VEX RT_BIT_32(28) /*< Indiciates VEX prefix. /
852	#define IEM_OP_PRF_EVEX RT_BIT_32(29) /*< Indiciates EVEX prefix. /
853	#define IEM_OP_PRF_XOP RT_BIT_32(30) /*< Indiciates XOP prefix. /
854	/** @} */
855
856	/** @name IEMOPFORM_XXX - Opcode forms
857	* @note These are ORed together with IEMOPHINT_XXX.
858	* @{ */
859	/** ModR/M: reg, r/m */
860	#define IEMOPFORM_RM 0
861	/** ModR/M: reg, r/m (register) */
862	#define IEMOPFORM_RM_REG (IEMOPFORM_RM \| IEMOPFORM_MOD3)
863	/** ModR/M: reg, r/m (memory) */
864	#define IEMOPFORM_RM_MEM (IEMOPFORM_RM \| IEMOPFORM_NOT_MOD3)
865	/** ModR/M: r/m, reg */
866	#define IEMOPFORM_MR 1
867	/** ModR/M: r/m (register), reg */
868	#define IEMOPFORM_MR_REG (IEMOPFORM_MR \| IEMOPFORM_MOD3)
869	/** ModR/M: r/m (memory), reg */
870	#define IEMOPFORM_MR_MEM (IEMOPFORM_MR \| IEMOPFORM_NOT_MOD3)
871	/** ModR/M: r/m only */
872	#define IEMOPFORM_M 2
873	/** ModR/M: r/m only (register). */
874	#define IEMOPFORM_M_REG (IEMOPFORM_M \| IEMOPFORM_MOD3)
875	/** ModR/M: r/m only (memory). */
876	#define IEMOPFORM_M_MEM (IEMOPFORM_M \| IEMOPFORM_NOT_MOD3)
877	/** ModR/M: reg only */
878	#define IEMOPFORM_R 3
879
880	/** VEX+ModR/M: reg, r/m */
881	#define IEMOPFORM_VEX_RM 4
882	/** VEX+ModR/M: reg, r/m (register) */
883	#define IEMOPFORM_VEX_RM_REG (IEMOPFORM_VEX_RM \| IEMOPFORM_MOD3)
884	/** VEX+ModR/M: reg, r/m (memory) */
885	#define IEMOPFORM_VEX_RM_MEM (IEMOPFORM_VEX_RM \| IEMOPFORM_NOT_MOD3)
886	/** VEX+ModR/M: r/m, reg */
887	#define IEMOPFORM_VEX_MR 5
888	/** VEX+ModR/M: r/m (register), reg */
889	#define IEMOPFORM_VEX_MR_REG (IEMOPFORM_VEX_MR \| IEMOPFORM_MOD3)
890	/** VEX+ModR/M: r/m (memory), reg */
891	#define IEMOPFORM_VEX_MR_MEM (IEMOPFORM_VEX_MR \| IEMOPFORM_NOT_MOD3)
892	/** VEX+ModR/M: r/m only */
893	#define IEMOPFORM_VEX_M 6
894	/** VEX+ModR/M: r/m only (register). */
895	#define IEMOPFORM_VEX_M_REG (IEMOPFORM_VEX_M \| IEMOPFORM_MOD3)
896	/** VEX+ModR/M: r/m only (memory). */
897	#define IEMOPFORM_VEX_M_MEM (IEMOPFORM_VEX_M \| IEMOPFORM_NOT_MOD3)
898	/** VEX+ModR/M: reg only */
899	#define IEMOPFORM_VEX_R 7
900	/** VEX+ModR/M: reg, vvvv, r/m */
901	#define IEMOPFORM_VEX_RVM 8
902	/** VEX+ModR/M: reg, vvvv, r/m (register). */
903	#define IEMOPFORM_VEX_RVM_REG (IEMOPFORM_VEX_RVM \| IEMOPFORM_MOD3)
904	/** VEX+ModR/M: reg, vvvv, r/m (memory). */
905	#define IEMOPFORM_VEX_RVM_MEM (IEMOPFORM_VEX_RVM \| IEMOPFORM_NOT_MOD3)
906	/** VEX+ModR/M: r/m, vvvv, reg */
907	#define IEMOPFORM_VEX_MVR 9
908	/** VEX+ModR/M: r/m, vvvv, reg (register) */
909	#define IEMOPFORM_VEX_MVR_REG (IEMOPFORM_VEX_MVR \| IEMOPFORM_MOD3)
910	/** VEX+ModR/M: r/m, vvvv, reg (memory) */
911	#define IEMOPFORM_VEX_MVR_MEM (IEMOPFORM_VEX_MVR \| IEMOPFORM_NOT_MOD3)
912
913	/** Fixed register instruction, no R/M. */
914	#define IEMOPFORM_FIXED 16
915
916	/** The r/m is a register. */
917	#define IEMOPFORM_MOD3 RT_BIT_32(8)
918	/** The r/m is a memory access. */
919	#define IEMOPFORM_NOT_MOD3 RT_BIT_32(9)
920	/** @} */
921
922	/** @name IEMOPHINT_XXX - Additional Opcode Hints
923	* @note These are ORed together with IEMOPFORM_XXX.
924	* @{ */
925	/** Ignores the operand size prefix (66h). */
926	#define IEMOPHINT_IGNORES_OZ_PFX RT_BIT_32(10)
927	/** Ignores REX.W (aka WIG). */
928	#define IEMOPHINT_IGNORES_REXW RT_BIT_32(11)
929	/** Both the operand size prefixes (66h + REX.W) are ignored. */
930	#define IEMOPHINT_IGNORES_OP_SIZES (IEMOPHINT_IGNORES_OZ_PFX \| IEMOPHINT_IGNORES_REXW)
931	/** Allowed with the lock prefix. */
932	#define IEMOPHINT_LOCK_ALLOWED RT_BIT_32(11)
933	/** The VEX.L value is ignored (aka LIG). */
934	#define IEMOPHINT_VEX_L_IGNORED RT_BIT_32(12)
935	/** The VEX.L value must be zero (i.e. 128-bit width only). */
936	#define IEMOPHINT_VEX_L_ZERO RT_BIT_32(13)
937
938	/** Hint to IEMAllInstructionPython.py that this macro should be skipped. */
939	#define IEMOPHINT_SKIP_PYTHON RT_BIT_32(31)
940	/** @} */
941
942	/**
943	* Possible hardware task switch sources.
944	*/
945	typedef enum IEMTASKSWITCH
946	{
947	/** Task switch caused by an interrupt/exception. */
948	IEMTASKSWITCH_INT_XCPT = 1,
949	/** Task switch caused by a far CALL. */
950	IEMTASKSWITCH_CALL,
951	/** Task switch caused by a far JMP. */
952	IEMTASKSWITCH_JUMP,
953	/** Task switch caused by an IRET. */
954	IEMTASKSWITCH_IRET
955	} IEMTASKSWITCH;
956	AssertCompileSize(IEMTASKSWITCH, 4);
957
958	/**
959	* Possible CrX load (write) sources.
960	*/
961	typedef enum IEMACCESSCRX
962	{
963	/** CrX access caused by 'mov crX' instruction. */
964	IEMACCESSCRX_MOV_CRX,
965	/** CrX (CR0) write caused by 'lmsw' instruction. */
966	IEMACCESSCRX_LMSW,
967	/** CrX (CR0) write caused by 'clts' instruction. */
968	IEMACCESSCRX_CLTS,
969	/** CrX (CR0) read caused by 'smsw' instruction. */
970	IEMACCESSCRX_SMSW
971	} IEMACCESSCRX;
972
973	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
974	/** @name IEM_SLAT_FAIL_XXX - Second-level address translation failure information.
975	*
976	* These flags provide further context to SLAT page-walk failures that could not be
977	* determined by PGM (e.g, PGM is not privy to memory access permissions).
978	*
979	* @{
980	*/
981	/** Translating a nested-guest linear address failed accessing a nested-guest
982	* physical address. */
983	# define IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR RT_BIT_32(0)
984	/** Translating a nested-guest linear address failed accessing a
985	* paging-structure entry or updating accessed/dirty bits. */
986	# define IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE RT_BIT_32(1)
987	/** @} */
988
989	PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) iemVmxApicAccessPageHandler;
990	# ifndef IN_RING3
991	DECLCALLBACK(FNPGMRZPHYSPFHANDLER) iemVmxApicAccessPagePfHandler;
992	# endif
993	#endif
994
995	/**
996	* Indicates to the verifier that the given flag set is undefined.
997	*
998	* Can be invoked again to add more flags.
999	*
1000	* This is a NOOP if the verifier isn't compiled in.
1001	*
1002	* @note We're temporarily keeping this until code is converted to new
1003	* disassembler style opcode handling.
1004	*/
1005	#define IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fEfl) do { } while (0)
1006
1007
1008	/** @def IEM_DECL_IMPL_TYPE
1009	* For typedef'ing an instruction implementation function.
1010	*
1011	* @param a_RetType The return type.
1012	* @param a_Name The name of the type.
1013	* @param a_ArgList The argument list enclosed in parentheses.
1014	*/
1015
1016	/** @def IEM_DECL_IMPL_DEF
1017	* For defining an instruction implementation function.
1018	*
1019	* @param a_RetType The return type.
1020	* @param a_Name The name of the type.
1021	* @param a_ArgList The argument list enclosed in parentheses.
1022	*/
1023
1024	#if defined(__GNUC__) && defined(RT_ARCH_X86)
1025	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1026	__attribute__((__fastcall__)) a_RetType (a_Name) a_ArgList
1027	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1028	__attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1029	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1030	__attribute__((__fastcall__, __nothrow__)) a_RetType a_Name a_ArgList
1031
1032	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
1033	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1034	a_RetType (__fastcall a_Name) a_ArgList
1035	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1036	a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1037	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1038	a_RetType __fastcall a_Name a_ArgList RT_NOEXCEPT
1039
1040	#elif __cplusplus >= 201700 /* P0012R1 support */
1041	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1042	a_RetType (VBOXCALL a_Name) a_ArgList RT_NOEXCEPT
1043	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1044	a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1045	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1046	a_RetType VBOXCALL a_Name a_ArgList RT_NOEXCEPT
1047
1048	#else
1049	# define IEM_DECL_IMPL_TYPE(a_RetType, a_Name, a_ArgList) \
1050	a_RetType (VBOXCALL a_Name) a_ArgList
1051	# define IEM_DECL_IMPL_DEF(a_RetType, a_Name, a_ArgList) \
1052	a_RetType VBOXCALL a_Name a_ArgList
1053	# define IEM_DECL_IMPL_PROTO(a_RetType, a_Name, a_ArgList) \
1054	a_RetType VBOXCALL a_Name a_ArgList
1055
1056	#endif
1057
1058	/** Defined in IEMAllAImplC.cpp but also used by IEMAllAImplA.asm. */
1059	RT_C_DECLS_BEGIN
1060	extern uint8_t const g_afParity[256];
1061	RT_C_DECLS_END
1062
1063
1064	/** @name Arithmetic assignment operations on bytes (binary).
1065	* @{ */
1066	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU8, (uint8_t pu8Dst, uint8_t u8Src, uint32_t pEFlags));
1067	typedef FNIEMAIMPLBINU8 *PFNIEMAIMPLBINU8;
1068	FNIEMAIMPLBINU8 iemAImpl_add_u8, iemAImpl_add_u8_locked;
1069	FNIEMAIMPLBINU8 iemAImpl_adc_u8, iemAImpl_adc_u8_locked;
1070	FNIEMAIMPLBINU8 iemAImpl_sub_u8, iemAImpl_sub_u8_locked;
1071	FNIEMAIMPLBINU8 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked;
1072	FNIEMAIMPLBINU8 iemAImpl_or_u8, iemAImpl_or_u8_locked;
1073	FNIEMAIMPLBINU8 iemAImpl_xor_u8, iemAImpl_xor_u8_locked;
1074	FNIEMAIMPLBINU8 iemAImpl_and_u8, iemAImpl_and_u8_locked;
1075	/** @} */
1076
1077	/** @name Arithmetic assignment operations on words (binary).
1078	* @{ */
1079	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU16, (uint16_t pu16Dst, uint16_t u16Src, uint32_t pEFlags));
1080	typedef FNIEMAIMPLBINU16 *PFNIEMAIMPLBINU16;
1081	FNIEMAIMPLBINU16 iemAImpl_add_u16, iemAImpl_add_u16_locked;
1082	FNIEMAIMPLBINU16 iemAImpl_adc_u16, iemAImpl_adc_u16_locked;
1083	FNIEMAIMPLBINU16 iemAImpl_sub_u16, iemAImpl_sub_u16_locked;
1084	FNIEMAIMPLBINU16 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked;
1085	FNIEMAIMPLBINU16 iemAImpl_or_u16, iemAImpl_or_u16_locked;
1086	FNIEMAIMPLBINU16 iemAImpl_xor_u16, iemAImpl_xor_u16_locked;
1087	FNIEMAIMPLBINU16 iemAImpl_and_u16, iemAImpl_and_u16_locked;
1088	/** @} */
1089
1090	/** @name Arithmetic assignment operations on double words (binary).
1091	* @{ */
1092	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU32, (uint32_t pu32Dst, uint32_t u32Src, uint32_t pEFlags));
1093	typedef FNIEMAIMPLBINU32 *PFNIEMAIMPLBINU32;
1094	FNIEMAIMPLBINU32 iemAImpl_add_u32, iemAImpl_add_u32_locked;
1095	FNIEMAIMPLBINU32 iemAImpl_adc_u32, iemAImpl_adc_u32_locked;
1096	FNIEMAIMPLBINU32 iemAImpl_sub_u32, iemAImpl_sub_u32_locked;
1097	FNIEMAIMPLBINU32 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked;
1098	FNIEMAIMPLBINU32 iemAImpl_or_u32, iemAImpl_or_u32_locked;
1099	FNIEMAIMPLBINU32 iemAImpl_xor_u32, iemAImpl_xor_u32_locked;
1100	FNIEMAIMPLBINU32 iemAImpl_and_u32, iemAImpl_and_u32_locked;
1101	/** @} */
1102
1103	/** @name Arithmetic assignment operations on quad words (binary).
1104	* @{ */
1105	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLBINU64, (uint64_t pu64Dst, uint64_t u64Src, uint32_t pEFlags));
1106	typedef FNIEMAIMPLBINU64 *PFNIEMAIMPLBINU64;
1107	FNIEMAIMPLBINU64 iemAImpl_add_u64, iemAImpl_add_u64_locked;
1108	FNIEMAIMPLBINU64 iemAImpl_adc_u64, iemAImpl_adc_u64_locked;
1109	FNIEMAIMPLBINU64 iemAImpl_sub_u64, iemAImpl_sub_u64_locked;
1110	FNIEMAIMPLBINU64 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked;
1111	FNIEMAIMPLBINU64 iemAImpl_or_u64, iemAImpl_or_u64_locked;
1112	FNIEMAIMPLBINU64 iemAImpl_xor_u64, iemAImpl_xor_u64_locked;
1113	FNIEMAIMPLBINU64 iemAImpl_and_u64, iemAImpl_and_u64_locked;
1114	/** @} */
1115
1116	/** @name Compare operations (thrown in with the binary ops).
1117	* @{ */
1118	FNIEMAIMPLBINU8 iemAImpl_cmp_u8;
1119	FNIEMAIMPLBINU16 iemAImpl_cmp_u16;
1120	FNIEMAIMPLBINU32 iemAImpl_cmp_u32;
1121	FNIEMAIMPLBINU64 iemAImpl_cmp_u64;
1122	/** @} */
1123
1124	/** @name Test operations (thrown in with the binary ops).
1125	* @{ */
1126	FNIEMAIMPLBINU8 iemAImpl_test_u8;
1127	FNIEMAIMPLBINU16 iemAImpl_test_u16;
1128	FNIEMAIMPLBINU32 iemAImpl_test_u32;
1129	FNIEMAIMPLBINU64 iemAImpl_test_u64;
1130	/** @} */
1131
1132	/** @name Bit operations operations (thrown in with the binary ops).
1133	* @{ */
1134	FNIEMAIMPLBINU16 iemAImpl_bt_u16;
1135	FNIEMAIMPLBINU32 iemAImpl_bt_u32;
1136	FNIEMAIMPLBINU64 iemAImpl_bt_u64;
1137	FNIEMAIMPLBINU16 iemAImpl_btc_u16, iemAImpl_btc_u16_locked;
1138	FNIEMAIMPLBINU32 iemAImpl_btc_u32, iemAImpl_btc_u32_locked;
1139	FNIEMAIMPLBINU64 iemAImpl_btc_u64, iemAImpl_btc_u64_locked;
1140	FNIEMAIMPLBINU16 iemAImpl_btr_u16, iemAImpl_btr_u16_locked;
1141	FNIEMAIMPLBINU32 iemAImpl_btr_u32, iemAImpl_btr_u32_locked;
1142	FNIEMAIMPLBINU64 iemAImpl_btr_u64, iemAImpl_btr_u64_locked;
1143	FNIEMAIMPLBINU16 iemAImpl_bts_u16, iemAImpl_bts_u16_locked;
1144	FNIEMAIMPLBINU32 iemAImpl_bts_u32, iemAImpl_bts_u32_locked;
1145	FNIEMAIMPLBINU64 iemAImpl_bts_u64, iemAImpl_bts_u64_locked;
1146	/** @} */
1147
1148	/** @name Exchange memory with register operations.
1149	* @{ */
1150	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked, (uint8_t pu8Mem, uint8_t pu8Reg));
1151	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t pu16Mem, uint16_t pu16Reg));
1152	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t pu32Mem, uint32_t pu32Reg));
1153	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t pu64Mem, uint64_t pu64Reg));
1154	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked, (uint8_t pu8Mem, uint8_t pu8Reg));
1155	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t pu16Mem, uint16_t pu16Reg));
1156	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t pu32Mem, uint32_t pu32Reg));
1157	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t pu64Mem, uint64_t pu64Reg));
1158	/** @} */
1159
1160	/** @name Exchange and add operations.
1161	* @{ */
1162	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8, (uint8_t pu8Dst, uint8_t pu8Reg, uint32_t *pEFlags));
1163	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16,(uint16_t pu16Dst, uint16_t pu16Reg, uint32_t *pEFlags));
1164	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32,(uint32_t pu32Dst, uint32_t pu32Reg, uint32_t *pEFlags));
1165	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64,(uint64_t pu64Dst, uint64_t pu64Reg, uint32_t *pEFlags));
1166	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u8_locked, (uint8_t pu8Dst, uint8_t pu8Reg, uint32_t *pEFlags));
1167	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u16_locked,(uint16_t pu16Dst, uint16_t pu16Reg, uint32_t *pEFlags));
1168	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u32_locked,(uint32_t pu32Dst, uint32_t pu32Reg, uint32_t *pEFlags));
1169	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u64_locked,(uint64_t pu64Dst, uint64_t pu64Reg, uint32_t *pEFlags));
1170	/** @} */
1171
1172	/** @name Compare and exchange.
1173	* @{ */
1174	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t pu8Dst, uint8_t puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1175	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t pu8Dst, uint8_t puAl, uint8_t uSrcReg, uint32_t *pEFlags));
1176	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t pu16Dst, uint16_t puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1177	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t pu16Dst, uint16_t puAx, uint16_t uSrcReg, uint32_t *pEFlags));
1178	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t pu32Dst, uint32_t puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1179	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t pu32Dst, uint32_t puEax, uint32_t uSrcReg, uint32_t *pEFlags));
1180	#if ARCH_BITS == 32
1181	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t pu64Dst, uint64_t puRax, uint64_t puSrcReg, uint32_t pEFlags));
1182	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t pu64Dst, uint64_t puRax, uint64_t puSrcReg, uint32_t pEFlags));
1183	#else
1184	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t pu64Dst, uint64_t puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1185	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t pu64Dst, uint64_t puRax, uint64_t uSrcReg, uint32_t *pEFlags));
1186	#endif
1187	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1188	uint32_t *pEFlags));
1189	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1190	uint32_t *pEFlags));
1191	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1192	uint32_t *pEFlags));
1193	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1194	uint32_t *pEFlags));
1195	#ifndef RT_ARCH_ARM64
1196	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
1197	PRTUINT128U pu128RbxRcx, uint32_t *pEFlags));
1198	#endif
1199	/** @} */
1200
1201	/** @name Memory ordering
1202	* @{ */
1203	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEMFENCE,(void));
1204	typedef FNIEMAIMPLMEMFENCE *PFNIEMAIMPLMEMFENCE;
1205	IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void));
1206	IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void));
1207	IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void));
1208	#ifndef RT_ARCH_ARM64
1209	IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void));
1210	#endif
1211	/** @} */
1212
1213	/** @name Double precision shifts
1214	* @{ */
1215	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU16,(uint16_t pu16Dst, uint16_t u16Src, uint8_t cShift, uint32_t pEFlags));
1216	typedef FNIEMAIMPLSHIFTDBLU16 *PFNIEMAIMPLSHIFTDBLU16;
1217	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU32,(uint32_t pu32Dst, uint32_t u32Src, uint8_t cShift, uint32_t pEFlags));
1218	typedef FNIEMAIMPLSHIFTDBLU32 *PFNIEMAIMPLSHIFTDBLU32;
1219	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTDBLU64,(uint64_t pu64Dst, uint64_t u64Src, uint8_t cShift, uint32_t pEFlags));
1220	typedef FNIEMAIMPLSHIFTDBLU64 *PFNIEMAIMPLSHIFTDBLU64;
1221	FNIEMAIMPLSHIFTDBLU16 iemAImpl_shld_u16, iemAImpl_shld_u16_amd, iemAImpl_shld_u16_intel;
1222	FNIEMAIMPLSHIFTDBLU32 iemAImpl_shld_u32, iemAImpl_shld_u32_amd, iemAImpl_shld_u32_intel;
1223	FNIEMAIMPLSHIFTDBLU64 iemAImpl_shld_u64, iemAImpl_shld_u64_amd, iemAImpl_shld_u64_intel;
1224	FNIEMAIMPLSHIFTDBLU16 iemAImpl_shrd_u16, iemAImpl_shrd_u16_amd, iemAImpl_shrd_u16_intel;
1225	FNIEMAIMPLSHIFTDBLU32 iemAImpl_shrd_u32, iemAImpl_shrd_u32_amd, iemAImpl_shrd_u32_intel;
1226	FNIEMAIMPLSHIFTDBLU64 iemAImpl_shrd_u64, iemAImpl_shrd_u64_amd, iemAImpl_shrd_u64_intel;
1227	/** @} */
1228
1229
1230	/** @name Bit search operations (thrown in with the binary ops).
1231	* @{ */
1232	FNIEMAIMPLBINU16 iemAImpl_bsf_u16, iemAImpl_bsf_u16_amd, iemAImpl_bsf_u16_intel;
1233	FNIEMAIMPLBINU32 iemAImpl_bsf_u32, iemAImpl_bsf_u32_amd, iemAImpl_bsf_u32_intel;
1234	FNIEMAIMPLBINU64 iemAImpl_bsf_u64, iemAImpl_bsf_u64_amd, iemAImpl_bsf_u64_intel;
1235	FNIEMAIMPLBINU16 iemAImpl_bsr_u16, iemAImpl_bsr_u16_amd, iemAImpl_bsr_u16_intel;
1236	FNIEMAIMPLBINU32 iemAImpl_bsr_u32, iemAImpl_bsr_u32_amd, iemAImpl_bsr_u32_intel;
1237	FNIEMAIMPLBINU64 iemAImpl_bsr_u64, iemAImpl_bsr_u64_amd, iemAImpl_bsr_u64_intel;
1238	/** @} */
1239
1240	/** @name Signed multiplication operations (thrown in with the binary ops).
1241	* @{ */
1242	FNIEMAIMPLBINU16 iemAImpl_imul_two_u16, iemAImpl_imul_two_u16_amd, iemAImpl_imul_two_u16_intel;
1243	FNIEMAIMPLBINU32 iemAImpl_imul_two_u32, iemAImpl_imul_two_u32_amd, iemAImpl_imul_two_u32_intel;
1244	FNIEMAIMPLBINU64 iemAImpl_imul_two_u64, iemAImpl_imul_two_u64_amd, iemAImpl_imul_two_u64_intel;
1245	/** @} */
1246
1247	/** @name Arithmetic assignment operations on bytes (unary).
1248	* @{ */
1249	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU8, (uint8_t pu8Dst, uint32_t pEFlags));
1250	typedef FNIEMAIMPLUNARYU8 *PFNIEMAIMPLUNARYU8;
1251	FNIEMAIMPLUNARYU8 iemAImpl_inc_u8, iemAImpl_inc_u8_locked;
1252	FNIEMAIMPLUNARYU8 iemAImpl_dec_u8, iemAImpl_dec_u8_locked;
1253	FNIEMAIMPLUNARYU8 iemAImpl_not_u8, iemAImpl_not_u8_locked;
1254	FNIEMAIMPLUNARYU8 iemAImpl_neg_u8, iemAImpl_neg_u8_locked;
1255	/** @} */
1256
1257	/** @name Arithmetic assignment operations on words (unary).
1258	* @{ */
1259	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU16, (uint16_t pu16Dst, uint32_t pEFlags));
1260	typedef FNIEMAIMPLUNARYU16 *PFNIEMAIMPLUNARYU16;
1261	FNIEMAIMPLUNARYU16 iemAImpl_inc_u16, iemAImpl_inc_u16_locked;
1262	FNIEMAIMPLUNARYU16 iemAImpl_dec_u16, iemAImpl_dec_u16_locked;
1263	FNIEMAIMPLUNARYU16 iemAImpl_not_u16, iemAImpl_not_u16_locked;
1264	FNIEMAIMPLUNARYU16 iemAImpl_neg_u16, iemAImpl_neg_u16_locked;
1265	/** @} */
1266
1267	/** @name Arithmetic assignment operations on double words (unary).
1268	* @{ */
1269	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU32, (uint32_t pu32Dst, uint32_t pEFlags));
1270	typedef FNIEMAIMPLUNARYU32 *PFNIEMAIMPLUNARYU32;
1271	FNIEMAIMPLUNARYU32 iemAImpl_inc_u32, iemAImpl_inc_u32_locked;
1272	FNIEMAIMPLUNARYU32 iemAImpl_dec_u32, iemAImpl_dec_u32_locked;
1273	FNIEMAIMPLUNARYU32 iemAImpl_not_u32, iemAImpl_not_u32_locked;
1274	FNIEMAIMPLUNARYU32 iemAImpl_neg_u32, iemAImpl_neg_u32_locked;
1275	/** @} */
1276
1277	/** @name Arithmetic assignment operations on quad words (unary).
1278	* @{ */
1279	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLUNARYU64, (uint64_t pu64Dst, uint32_t pEFlags));
1280	typedef FNIEMAIMPLUNARYU64 *PFNIEMAIMPLUNARYU64;
1281	FNIEMAIMPLUNARYU64 iemAImpl_inc_u64, iemAImpl_inc_u64_locked;
1282	FNIEMAIMPLUNARYU64 iemAImpl_dec_u64, iemAImpl_dec_u64_locked;
1283	FNIEMAIMPLUNARYU64 iemAImpl_not_u64, iemAImpl_not_u64_locked;
1284	FNIEMAIMPLUNARYU64 iemAImpl_neg_u64, iemAImpl_neg_u64_locked;
1285	/** @} */
1286
1287
1288	/** @name Shift operations on bytes (Group 2).
1289	* @{ */
1290	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU8,(uint8_t pu8Dst, uint8_t cShift, uint32_t pEFlags));
1291	typedef FNIEMAIMPLSHIFTU8 *PFNIEMAIMPLSHIFTU8;
1292	FNIEMAIMPLSHIFTU8 iemAImpl_rol_u8, iemAImpl_rol_u8_amd, iemAImpl_rol_u8_intel;
1293	FNIEMAIMPLSHIFTU8 iemAImpl_ror_u8, iemAImpl_ror_u8_amd, iemAImpl_ror_u8_intel;
1294	FNIEMAIMPLSHIFTU8 iemAImpl_rcl_u8, iemAImpl_rcl_u8_amd, iemAImpl_rcl_u8_intel;
1295	FNIEMAIMPLSHIFTU8 iemAImpl_rcr_u8, iemAImpl_rcr_u8_amd, iemAImpl_rcr_u8_intel;
1296	FNIEMAIMPLSHIFTU8 iemAImpl_shl_u8, iemAImpl_shl_u8_amd, iemAImpl_shl_u8_intel;
1297	FNIEMAIMPLSHIFTU8 iemAImpl_shr_u8, iemAImpl_shr_u8_amd, iemAImpl_shr_u8_intel;
1298	FNIEMAIMPLSHIFTU8 iemAImpl_sar_u8, iemAImpl_sar_u8_amd, iemAImpl_sar_u8_intel;
1299	/** @} */
1300
1301	/** @name Shift operations on words (Group 2).
1302	* @{ */
1303	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU16,(uint16_t pu16Dst, uint8_t cShift, uint32_t pEFlags));
1304	typedef FNIEMAIMPLSHIFTU16 *PFNIEMAIMPLSHIFTU16;
1305	FNIEMAIMPLSHIFTU16 iemAImpl_rol_u16, iemAImpl_rol_u16_amd, iemAImpl_rol_u16_intel;
1306	FNIEMAIMPLSHIFTU16 iemAImpl_ror_u16, iemAImpl_ror_u16_amd, iemAImpl_ror_u16_intel;
1307	FNIEMAIMPLSHIFTU16 iemAImpl_rcl_u16, iemAImpl_rcl_u16_amd, iemAImpl_rcl_u16_intel;
1308	FNIEMAIMPLSHIFTU16 iemAImpl_rcr_u16, iemAImpl_rcr_u16_amd, iemAImpl_rcr_u16_intel;
1309	FNIEMAIMPLSHIFTU16 iemAImpl_shl_u16, iemAImpl_shl_u16_amd, iemAImpl_shl_u16_intel;
1310	FNIEMAIMPLSHIFTU16 iemAImpl_shr_u16, iemAImpl_shr_u16_amd, iemAImpl_shr_u16_intel;
1311	FNIEMAIMPLSHIFTU16 iemAImpl_sar_u16, iemAImpl_sar_u16_amd, iemAImpl_sar_u16_intel;
1312	/** @} */
1313
1314	/** @name Shift operations on double words (Group 2).
1315	* @{ */
1316	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU32,(uint32_t pu32Dst, uint8_t cShift, uint32_t pEFlags));
1317	typedef FNIEMAIMPLSHIFTU32 *PFNIEMAIMPLSHIFTU32;
1318	FNIEMAIMPLSHIFTU32 iemAImpl_rol_u32, iemAImpl_rol_u32_amd, iemAImpl_rol_u32_intel;
1319	FNIEMAIMPLSHIFTU32 iemAImpl_ror_u32, iemAImpl_ror_u32_amd, iemAImpl_ror_u32_intel;
1320	FNIEMAIMPLSHIFTU32 iemAImpl_rcl_u32, iemAImpl_rcl_u32_amd, iemAImpl_rcl_u32_intel;
1321	FNIEMAIMPLSHIFTU32 iemAImpl_rcr_u32, iemAImpl_rcr_u32_amd, iemAImpl_rcr_u32_intel;
1322	FNIEMAIMPLSHIFTU32 iemAImpl_shl_u32, iemAImpl_shl_u32_amd, iemAImpl_shl_u32_intel;
1323	FNIEMAIMPLSHIFTU32 iemAImpl_shr_u32, iemAImpl_shr_u32_amd, iemAImpl_shr_u32_intel;
1324	FNIEMAIMPLSHIFTU32 iemAImpl_sar_u32, iemAImpl_sar_u32_amd, iemAImpl_sar_u32_intel;
1325	/** @} */
1326
1327	/** @name Shift operations on words (Group 2).
1328	* @{ */
1329	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLSHIFTU64,(uint64_t pu64Dst, uint8_t cShift, uint32_t pEFlags));
1330	typedef FNIEMAIMPLSHIFTU64 *PFNIEMAIMPLSHIFTU64;
1331	FNIEMAIMPLSHIFTU64 iemAImpl_rol_u64, iemAImpl_rol_u64_amd, iemAImpl_rol_u64_intel;
1332	FNIEMAIMPLSHIFTU64 iemAImpl_ror_u64, iemAImpl_ror_u64_amd, iemAImpl_ror_u64_intel;
1333	FNIEMAIMPLSHIFTU64 iemAImpl_rcl_u64, iemAImpl_rcl_u64_amd, iemAImpl_rcl_u64_intel;
1334	FNIEMAIMPLSHIFTU64 iemAImpl_rcr_u64, iemAImpl_rcr_u64_amd, iemAImpl_rcr_u64_intel;
1335	FNIEMAIMPLSHIFTU64 iemAImpl_shl_u64, iemAImpl_shl_u64_amd, iemAImpl_shl_u64_intel;
1336	FNIEMAIMPLSHIFTU64 iemAImpl_shr_u64, iemAImpl_shr_u64_amd, iemAImpl_shr_u64_intel;
1337	FNIEMAIMPLSHIFTU64 iemAImpl_sar_u64, iemAImpl_sar_u64_amd, iemAImpl_sar_u64_intel;
1338	/** @} */
1339
1340	/** @name Multiplication and division operations.
1341	* @{ */
1342	typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU8,(uint16_t pu16AX, uint8_t u8FactorDivisor, uint32_t pEFlags));
1343	typedef FNIEMAIMPLMULDIVU8 *PFNIEMAIMPLMULDIVU8;
1344	FNIEMAIMPLMULDIVU8 iemAImpl_mul_u8, iemAImpl_mul_u8_amd, iemAImpl_mul_u8_intel;
1345	FNIEMAIMPLMULDIVU8 iemAImpl_imul_u8, iemAImpl_imul_u8_amd, iemAImpl_imul_u8_intel;
1346	FNIEMAIMPLMULDIVU8 iemAImpl_div_u8, iemAImpl_div_u8_amd, iemAImpl_div_u8_intel;
1347	FNIEMAIMPLMULDIVU8 iemAImpl_idiv_u8, iemAImpl_idiv_u8_amd, iemAImpl_idiv_u8_intel;
1348
1349	typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU16,(uint16_t pu16AX, uint16_t pu16DX, uint16_t u16FactorDivisor, uint32_t *pEFlags));
1350	typedef FNIEMAIMPLMULDIVU16 *PFNIEMAIMPLMULDIVU16;
1351	FNIEMAIMPLMULDIVU16 iemAImpl_mul_u16, iemAImpl_mul_u16_amd, iemAImpl_mul_u16_intel;
1352	FNIEMAIMPLMULDIVU16 iemAImpl_imul_u16, iemAImpl_imul_u16_amd, iemAImpl_imul_u16_intel;
1353	FNIEMAIMPLMULDIVU16 iemAImpl_div_u16, iemAImpl_div_u16_amd, iemAImpl_div_u16_intel;
1354	FNIEMAIMPLMULDIVU16 iemAImpl_idiv_u16, iemAImpl_idiv_u16_amd, iemAImpl_idiv_u16_intel;
1355
1356	typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU32,(uint32_t pu32EAX, uint32_t pu32EDX, uint32_t u32FactorDivisor, uint32_t *pEFlags));
1357	typedef FNIEMAIMPLMULDIVU32 *PFNIEMAIMPLMULDIVU32;
1358	FNIEMAIMPLMULDIVU32 iemAImpl_mul_u32, iemAImpl_mul_u32_amd, iemAImpl_mul_u32_intel;
1359	FNIEMAIMPLMULDIVU32 iemAImpl_imul_u32, iemAImpl_imul_u32_amd, iemAImpl_imul_u32_intel;
1360	FNIEMAIMPLMULDIVU32 iemAImpl_div_u32, iemAImpl_div_u32_amd, iemAImpl_div_u32_intel;
1361	FNIEMAIMPLMULDIVU32 iemAImpl_idiv_u32, iemAImpl_idiv_u32_amd, iemAImpl_idiv_u32_intel;
1362
1363	typedef IEM_DECL_IMPL_TYPE(int, FNIEMAIMPLMULDIVU64,(uint64_t pu64RAX, uint64_t pu64RDX, uint64_t u64FactorDivisor, uint32_t *pEFlags));
1364	typedef FNIEMAIMPLMULDIVU64 *PFNIEMAIMPLMULDIVU64;
1365	FNIEMAIMPLMULDIVU64 iemAImpl_mul_u64, iemAImpl_mul_u64_amd, iemAImpl_mul_u64_intel;
1366	FNIEMAIMPLMULDIVU64 iemAImpl_imul_u64, iemAImpl_imul_u64_amd, iemAImpl_imul_u64_intel;
1367	FNIEMAIMPLMULDIVU64 iemAImpl_div_u64, iemAImpl_div_u64_amd, iemAImpl_div_u64_intel;
1368	FNIEMAIMPLMULDIVU64 iemAImpl_idiv_u64, iemAImpl_idiv_u64_amd, iemAImpl_idiv_u64_intel;
1369	/** @} */
1370
1371	/** @name Byte Swap.
1372	* @{ */
1373	IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u16,(uint32_t pu32Dst)); / Yes, 32-bit register access. */
1374	IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u32,(uint32_t *pu32Dst));
1375	IEM_DECL_IMPL_TYPE(void, iemAImpl_bswap_u64,(uint64_t *pu64Dst));
1376	/** @} */
1377
1378	/** @name Misc.
1379	* @{ */
1380	FNIEMAIMPLBINU16 iemAImpl_arpl;
1381	/** @} */
1382
1383
1384	/** @name FPU operations taking a 32-bit float argument
1385	* @{ */
1386	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1387	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1388	typedef FNIEMAIMPLFPUR32FSW *PFNIEMAIMPLFPUR32FSW;
1389
1390	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1391	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2));
1392	typedef FNIEMAIMPLFPUR32 *PFNIEMAIMPLFPUR32;
1393
1394	FNIEMAIMPLFPUR32FSW iemAImpl_fcom_r80_by_r32;
1395	FNIEMAIMPLFPUR32 iemAImpl_fadd_r80_by_r32;
1396	FNIEMAIMPLFPUR32 iemAImpl_fmul_r80_by_r32;
1397	FNIEMAIMPLFPUR32 iemAImpl_fsub_r80_by_r32;
1398	FNIEMAIMPLFPUR32 iemAImpl_fsubr_r80_by_r32;
1399	FNIEMAIMPLFPUR32 iemAImpl_fdiv_r80_by_r32;
1400	FNIEMAIMPLFPUR32 iemAImpl_fdivr_r80_by_r32;
1401
1402	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val));
1403	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1404	PRTFLOAT32U pr32Val, PCRTFLOAT80U pr80Val));
1405	/** @} */
1406
1407	/** @name FPU operations taking a 64-bit float argument
1408	* @{ */
1409	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1410	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1411	typedef FNIEMAIMPLFPUR64FSW *PFNIEMAIMPLFPUR64FSW;
1412
1413	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1414	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2));
1415	typedef FNIEMAIMPLFPUR64 *PFNIEMAIMPLFPUR64;
1416
1417	FNIEMAIMPLFPUR64FSW iemAImpl_fcom_r80_by_r64;
1418	FNIEMAIMPLFPUR64 iemAImpl_fadd_r80_by_r64;
1419	FNIEMAIMPLFPUR64 iemAImpl_fmul_r80_by_r64;
1420	FNIEMAIMPLFPUR64 iemAImpl_fsub_r80_by_r64;
1421	FNIEMAIMPLFPUR64 iemAImpl_fsubr_r80_by_r64;
1422	FNIEMAIMPLFPUR64 iemAImpl_fdiv_r80_by_r64;
1423	FNIEMAIMPLFPUR64 iemAImpl_fdivr_r80_by_r64;
1424
1425	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val));
1426	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1427	PRTFLOAT64U pr32Val, PCRTFLOAT80U pr80Val));
1428	/** @} */
1429
1430	/** @name FPU operations taking a 80-bit float argument
1431	* @{ */
1432	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1433	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1434	typedef FNIEMAIMPLFPUR80 *PFNIEMAIMPLFPUR80;
1435	FNIEMAIMPLFPUR80 iemAImpl_fadd_r80_by_r80;
1436	FNIEMAIMPLFPUR80 iemAImpl_fmul_r80_by_r80;
1437	FNIEMAIMPLFPUR80 iemAImpl_fsub_r80_by_r80;
1438	FNIEMAIMPLFPUR80 iemAImpl_fsubr_r80_by_r80;
1439	FNIEMAIMPLFPUR80 iemAImpl_fdiv_r80_by_r80;
1440	FNIEMAIMPLFPUR80 iemAImpl_fdivr_r80_by_r80;
1441	FNIEMAIMPLFPUR80 iemAImpl_fprem_r80_by_r80;
1442	FNIEMAIMPLFPUR80 iemAImpl_fprem1_r80_by_r80;
1443	FNIEMAIMPLFPUR80 iemAImpl_fscale_r80_by_r80;
1444
1445	FNIEMAIMPLFPUR80 iemAImpl_fpatan_r80_by_r80, iemAImpl_fpatan_r80_by_r80_amd, iemAImpl_fpatan_r80_by_r80_intel;
1446	FNIEMAIMPLFPUR80 iemAImpl_fyl2x_r80_by_r80, iemAImpl_fyl2x_r80_by_r80_amd, iemAImpl_fyl2x_r80_by_r80_intel;
1447	FNIEMAIMPLFPUR80 iemAImpl_fyl2xp1_r80_by_r80, iemAImpl_fyl2xp1_r80_by_r80_amd, iemAImpl_fyl2xp1_r80_by_r80_intel;
1448
1449	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1450	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1451	typedef FNIEMAIMPLFPUR80FSW *PFNIEMAIMPLFPUR80FSW;
1452	FNIEMAIMPLFPUR80FSW iemAImpl_fcom_r80_by_r80;
1453	FNIEMAIMPLFPUR80FSW iemAImpl_fucom_r80_by_r80;
1454
1455	typedef IEM_DECL_IMPL_TYPE(uint32_t, FNIEMAIMPLFPUR80EFL,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
1456	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2));
1457	typedef FNIEMAIMPLFPUR80EFL *PFNIEMAIMPLFPUR80EFL;
1458	FNIEMAIMPLFPUR80EFL iemAImpl_fcomi_r80_by_r80;
1459	FNIEMAIMPLFPUR80EFL iemAImpl_fucomi_r80_by_r80;
1460
1461	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARY,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1462	typedef FNIEMAIMPLFPUR80UNARY *PFNIEMAIMPLFPUR80UNARY;
1463	FNIEMAIMPLFPUR80UNARY iemAImpl_fabs_r80;
1464	FNIEMAIMPLFPUR80UNARY iemAImpl_fchs_r80;
1465	FNIEMAIMPLFPUR80UNARY iemAImpl_f2xm1_r80, iemAImpl_f2xm1_r80_amd, iemAImpl_f2xm1_r80_intel;
1466	FNIEMAIMPLFPUR80UNARY iemAImpl_fsqrt_r80;
1467	FNIEMAIMPLFPUR80UNARY iemAImpl_frndint_r80;
1468	FNIEMAIMPLFPUR80UNARY iemAImpl_fsin_r80, iemAImpl_fsin_r80_amd, iemAImpl_fsin_r80_intel;
1469	FNIEMAIMPLFPUR80UNARY iemAImpl_fcos_r80, iemAImpl_fcos_r80_amd, iemAImpl_fcos_r80_intel;
1470
1471	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYFSW,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val));
1472	typedef FNIEMAIMPLFPUR80UNARYFSW *PFNIEMAIMPLFPUR80UNARYFSW;
1473	FNIEMAIMPLFPUR80UNARYFSW iemAImpl_ftst_r80;
1474	FNIEMAIMPLFPUR80UNARYFSW iemAImpl_fxam_r80;
1475
1476	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80LDCONST,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes));
1477	typedef FNIEMAIMPLFPUR80LDCONST *PFNIEMAIMPLFPUR80LDCONST;
1478	FNIEMAIMPLFPUR80LDCONST iemAImpl_fld1;
1479	FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2t;
1480	FNIEMAIMPLFPUR80LDCONST iemAImpl_fldl2e;
1481	FNIEMAIMPLFPUR80LDCONST iemAImpl_fldpi;
1482	FNIEMAIMPLFPUR80LDCONST iemAImpl_fldlg2;
1483	FNIEMAIMPLFPUR80LDCONST iemAImpl_fldln2;
1484	FNIEMAIMPLFPUR80LDCONST iemAImpl_fldz;
1485
1486	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUR80UNARYTWO,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo,
1487	PCRTFLOAT80U pr80Val));
1488	typedef FNIEMAIMPLFPUR80UNARYTWO *PFNIEMAIMPLFPUR80UNARYTWO;
1489	FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fptan_r80_r80, iemAImpl_fptan_r80_r80_amd, iemAImpl_fptan_r80_r80_intel;
1490	FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fxtract_r80_r80;
1491	FNIEMAIMPLFPUR80UNARYTWO iemAImpl_fsincos_r80_r80, iemAImpl_fsincos_r80_r80_amd, iemAImpl_fsincos_r80_r80_intel;
1492
1493	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val));
1494	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1495	PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src));
1496
1497	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val));
1498	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
1499	PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src));
1500
1501	/** @} */
1502
1503	/** @name FPU operations taking a 16-bit signed integer argument
1504	* @{ */
1505	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1506	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1507	typedef FNIEMAIMPLFPUI16 *PFNIEMAIMPLFPUI16;
1508	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI16,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1509	int16_t *pi16Dst, PCRTFLOAT80U pr80Src));
1510	typedef FNIEMAIMPLFPUSTR80TOI16 *PFNIEMAIMPLFPUSTR80TOI16;
1511
1512	FNIEMAIMPLFPUI16 iemAImpl_fiadd_r80_by_i16;
1513	FNIEMAIMPLFPUI16 iemAImpl_fimul_r80_by_i16;
1514	FNIEMAIMPLFPUI16 iemAImpl_fisub_r80_by_i16;
1515	FNIEMAIMPLFPUI16 iemAImpl_fisubr_r80_by_i16;
1516	FNIEMAIMPLFPUI16 iemAImpl_fidiv_r80_by_i16;
1517	FNIEMAIMPLFPUI16 iemAImpl_fidivr_r80_by_i16;
1518
1519	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI16FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1520	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2));
1521	typedef FNIEMAIMPLFPUI16FSW *PFNIEMAIMPLFPUI16FSW;
1522	FNIEMAIMPLFPUI16FSW iemAImpl_ficom_r80_by_i16;
1523
1524	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int16_t const *pi16Val));
1525	FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fist_r80_to_i16;
1526	FNIEMAIMPLFPUSTR80TOI16 iemAImpl_fistt_r80_to_i16, iemAImpl_fistt_r80_to_i16_amd, iemAImpl_fistt_r80_to_i16_intel;
1527	/** @} */
1528
1529	/** @name FPU operations taking a 32-bit signed integer argument
1530	* @{ */
1531	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
1532	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1533	typedef FNIEMAIMPLFPUI32 *PFNIEMAIMPLFPUI32;
1534	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI32,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1535	int32_t *pi32Dst, PCRTFLOAT80U pr80Src));
1536	typedef FNIEMAIMPLFPUSTR80TOI32 *PFNIEMAIMPLFPUSTR80TOI32;
1537
1538	FNIEMAIMPLFPUI32 iemAImpl_fiadd_r80_by_i32;
1539	FNIEMAIMPLFPUI32 iemAImpl_fimul_r80_by_i32;
1540	FNIEMAIMPLFPUI32 iemAImpl_fisub_r80_by_i32;
1541	FNIEMAIMPLFPUI32 iemAImpl_fisubr_r80_by_i32;
1542	FNIEMAIMPLFPUI32 iemAImpl_fidiv_r80_by_i32;
1543	FNIEMAIMPLFPUI32 iemAImpl_fidivr_r80_by_i32;
1544
1545	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUI32FSW,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
1546	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2));
1547	typedef FNIEMAIMPLFPUI32FSW *PFNIEMAIMPLFPUI32FSW;
1548	FNIEMAIMPLFPUI32FSW iemAImpl_ficom_r80_by_i32;
1549
1550	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int32_t const *pi32Val));
1551	FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fist_r80_to_i32;
1552	FNIEMAIMPLFPUSTR80TOI32 iemAImpl_fistt_r80_to_i32;
1553	/** @} */
1554
1555	/** @name FPU operations taking a 64-bit signed integer argument
1556	* @{ */
1557	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLFPUSTR80TOI64,(PCX86FXSTATE pFpuState, uint16_t *pFpuRes,
1558	int64_t *pi64Dst, PCRTFLOAT80U pr80Src));
1559	typedef FNIEMAIMPLFPUSTR80TOI64 *PFNIEMAIMPLFPUSTR80TOI64;
1560
1561	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int64_t const *pi64Val));
1562	FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fist_r80_to_i64;
1563	FNIEMAIMPLFPUSTR80TOI64 iemAImpl_fistt_r80_to_i64;
1564	/** @} */
1565
1566
1567	/** Temporary type representing a 256-bit vector register. */
1568	typedef struct { uint64_t au64[4]; } IEMVMM256;
1569	/** Temporary type pointing to a 256-bit vector register. */
1570	typedef IEMVMM256 *PIEMVMM256;
1571	/** Temporary type pointing to a const 256-bit vector register. */
1572	typedef IEMVMM256 *PCIEMVMM256;
1573
1574
1575	/** @name Media (SSE/MMX/AVX) operations: full1 + full2 -> full1.
1576	* @{ */
1577	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src));
1578	typedef FNIEMAIMPLMEDIAF2U64 *PFNIEMAIMPLMEDIAF2U64;
1579	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF2U128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src));
1580	typedef FNIEMAIMPLMEDIAF2U128 *PFNIEMAIMPLMEDIAF2U128;
1581	FNIEMAIMPLMEDIAF2U64 iemAImpl_pxor_u64, iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqd_u64;
1582	FNIEMAIMPLMEDIAF2U128 iemAImpl_pxor_u128, iemAImpl_pcmpeqb_u128, iemAImpl_pcmpeqw_u128, iemAImpl_pcmpeqd_u128;
1583	/** @} */
1584
1585	/** @name Media (SSE/MMX/AVX) operations: lowhalf1 + lowhalf1 -> full1.
1586	* @{ */
1587	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1L1U64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src));
1588	typedef FNIEMAIMPLMEDIAF1L1U64 *PFNIEMAIMPLMEDIAF1L1U64;
1589	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1L1U128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src));
1590	typedef FNIEMAIMPLMEDIAF1L1U128 *PFNIEMAIMPLMEDIAF1L1U128;
1591	FNIEMAIMPLMEDIAF1L1U64 iemAImpl_punpcklbw_u64, iemAImpl_punpcklwd_u64, iemAImpl_punpckldq_u64;
1592	FNIEMAIMPLMEDIAF1L1U128 iemAImpl_punpcklbw_u128, iemAImpl_punpcklwd_u128, iemAImpl_punpckldq_u128, iemAImpl_punpcklqdq_u128;
1593	/** @} */
1594
1595	/** @name Media (SSE/MMX/AVX) operations: hihalf1 + hihalf2 -> full1.
1596	* @{ */
1597	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1H1U64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src));
1598	typedef FNIEMAIMPLMEDIAF2U64 *PFNIEMAIMPLMEDIAF1H1U64;
1599	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAF1H1U128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src));
1600	typedef FNIEMAIMPLMEDIAF2U128 *PFNIEMAIMPLMEDIAF1H1U128;
1601	FNIEMAIMPLMEDIAF1H1U64 iemAImpl_punpckhbw_u64, iemAImpl_punpckhwd_u64, iemAImpl_punpckhdq_u64;
1602	FNIEMAIMPLMEDIAF1H1U128 iemAImpl_punpckhbw_u128, iemAImpl_punpckhwd_u128, iemAImpl_punpckhdq_u128, iemAImpl_punpckhqdq_u128;
1603	/** @} */
1604
1605	/** @name Media (SSE/MMX/AVX) operation: Packed Shuffle Stuff (evil)
1606	* @{ */
1607	typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLMEDIAPSHUF,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst,
1608	PCRTUINT128U pu128Src, uint8_t bEvil));
1609	typedef FNIEMAIMPLMEDIAPSHUF *PFNIEMAIMPLMEDIAPSHUF;
1610	FNIEMAIMPLMEDIAPSHUF iemAImpl_pshufhw, iemAImpl_pshuflw, iemAImpl_pshufd;
1611	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src, uint8_t bEvil));
1612	/** @} */
1613
1614	/** @name Media (SSE/MMX/AVX) operation: Move Byte Mask
1615	* @{ */
1616	IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src));
1617	IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, PCRTUINT128U pu128Src));
1618	/** @} */
1619
1620	/** @name Media (SSE/MMX/AVX) operation: Sort this later
1621	* @{ */
1622	IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
1623	IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc));
1624	IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, uint64_t uSrc));
1625
1626	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1627	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1628	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc));
1629	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc));
1630
1631	/** @} */
1632
1633
1634	/** @name Function tables.
1635	* @{
1636	*/
1637
1638	/**
1639	* Function table for a binary operator providing implementation based on
1640	* operand size.
1641	*/
1642	typedef struct IEMOPBINSIZES
1643	{
1644	PFNIEMAIMPLBINU8 pfnNormalU8, pfnLockedU8;
1645	PFNIEMAIMPLBINU16 pfnNormalU16, pfnLockedU16;
1646	PFNIEMAIMPLBINU32 pfnNormalU32, pfnLockedU32;
1647	PFNIEMAIMPLBINU64 pfnNormalU64, pfnLockedU64;
1648	} IEMOPBINSIZES;
1649	/** Pointer to a binary operator function table. */
1650	typedef IEMOPBINSIZES const *PCIEMOPBINSIZES;
1651
1652
1653	/**
1654	* Function table for a unary operator providing implementation based on
1655	* operand size.
1656	*/
1657	typedef struct IEMOPUNARYSIZES
1658	{
1659	PFNIEMAIMPLUNARYU8 pfnNormalU8, pfnLockedU8;
1660	PFNIEMAIMPLUNARYU16 pfnNormalU16, pfnLockedU16;
1661	PFNIEMAIMPLUNARYU32 pfnNormalU32, pfnLockedU32;
1662	PFNIEMAIMPLUNARYU64 pfnNormalU64, pfnLockedU64;
1663	} IEMOPUNARYSIZES;
1664	/** Pointer to a unary operator function table. */
1665	typedef IEMOPUNARYSIZES const *PCIEMOPUNARYSIZES;
1666
1667
1668	/**
1669	* Function table for a shift operator providing implementation based on
1670	* operand size.
1671	*/
1672	typedef struct IEMOPSHIFTSIZES
1673	{
1674	PFNIEMAIMPLSHIFTU8 pfnNormalU8;
1675	PFNIEMAIMPLSHIFTU16 pfnNormalU16;
1676	PFNIEMAIMPLSHIFTU32 pfnNormalU32;
1677	PFNIEMAIMPLSHIFTU64 pfnNormalU64;
1678	} IEMOPSHIFTSIZES;
1679	/** Pointer to a shift operator function table. */
1680	typedef IEMOPSHIFTSIZES const *PCIEMOPSHIFTSIZES;
1681
1682
1683	/**
1684	* Function table for a multiplication or division operation.
1685	*/
1686	typedef struct IEMOPMULDIVSIZES
1687	{
1688	PFNIEMAIMPLMULDIVU8 pfnU8;
1689	PFNIEMAIMPLMULDIVU16 pfnU16;
1690	PFNIEMAIMPLMULDIVU32 pfnU32;
1691	PFNIEMAIMPLMULDIVU64 pfnU64;
1692	} IEMOPMULDIVSIZES;
1693	/** Pointer to a multiplication or division operation function table. */
1694	typedef IEMOPMULDIVSIZES const *PCIEMOPMULDIVSIZES;
1695
1696
1697	/**
1698	* Function table for a double precision shift operator providing implementation
1699	* based on operand size.
1700	*/
1701	typedef struct IEMOPSHIFTDBLSIZES
1702	{
1703	PFNIEMAIMPLSHIFTDBLU16 pfnNormalU16;
1704	PFNIEMAIMPLSHIFTDBLU32 pfnNormalU32;
1705	PFNIEMAIMPLSHIFTDBLU64 pfnNormalU64;
1706	} IEMOPSHIFTDBLSIZES;
1707	/** Pointer to a double precision shift function table. */
1708	typedef IEMOPSHIFTDBLSIZES const *PCIEMOPSHIFTDBLSIZES;
1709
1710
1711	/**
1712	* Function table for media instruction taking two full sized media registers,
1713	* optionally the 2nd being a memory reference (only modifying the first op.)
1714	*/
1715	typedef struct IEMOPMEDIAF2
1716	{
1717	PFNIEMAIMPLMEDIAF2U64 pfnU64;
1718	PFNIEMAIMPLMEDIAF2U128 pfnU128;
1719	} IEMOPMEDIAF2;
1720	/** Pointer to a media operation function table for full sized ops. */
1721	typedef IEMOPMEDIAF2 const *PCIEMOPMEDIAF2;
1722
1723	/**
1724	* Function table for media instruction taking taking one full and one lower
1725	* half media register.
1726	*/
1727	typedef struct IEMOPMEDIAF1L1
1728	{
1729	PFNIEMAIMPLMEDIAF1L1U64 pfnU64;
1730	PFNIEMAIMPLMEDIAF1L1U128 pfnU128;
1731	} IEMOPMEDIAF1L1;
1732	/** Pointer to a media operation function table for lowhalf+lowhalf -> full. */
1733	typedef IEMOPMEDIAF1L1 const *PCIEMOPMEDIAF1L1;
1734
1735	/**
1736	* Function table for media instruction taking taking one full and one high half
1737	* media register.
1738	*/
1739	typedef struct IEMOPMEDIAF1H1
1740	{
1741	PFNIEMAIMPLMEDIAF1H1U64 pfnU64;
1742	PFNIEMAIMPLMEDIAF1H1U128 pfnU128;
1743	} IEMOPMEDIAF1H1;
1744	/** Pointer to a media operation function table for hihalf+hihalf -> full. */
1745	typedef IEMOPMEDIAF1H1 const *PCIEMOPMEDIAF1H1;
1746
1747
1748	/** @} */
1749
1750
1751	/** @name C instruction implementations for anything slightly complicated.
1752	* @{ */
1753
1754	/**
1755	* For typedef'ing or declaring a C instruction implementation function taking
1756	* no extra arguments.
1757	*
1758	* @param a_Name The name of the type.
1759	*/
1760	# define IEM_CIMPL_DECL_TYPE_0(a_Name) \
1761	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
1762	/**
1763	* For defining a C instruction implementation function taking no extra
1764	* arguments.
1765	*
1766	* @param a_Name The name of the function
1767	*/
1768	# define IEM_CIMPL_DEF_0(a_Name) \
1769	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
1770	/**
1771	* Prototype version of IEM_CIMPL_DEF_0.
1772	*/
1773	# define IEM_CIMPL_PROTO_0(a_Name) \
1774	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr))
1775	/**
1776	* For calling a C instruction implementation function taking no extra
1777	* arguments.
1778	*
1779	* This special call macro adds default arguments to the call and allow us to
1780	* change these later.
1781	*
1782	* @param a_fn The name of the function.
1783	*/
1784	# define IEM_CIMPL_CALL_0(a_fn) a_fn(pVCpu, cbInstr)
1785
1786	/**
1787	* For typedef'ing or declaring a C instruction implementation function taking
1788	* one extra argument.
1789	*
1790	* @param a_Name The name of the type.
1791	* @param a_Type0 The argument type.
1792	* @param a_Arg0 The argument name.
1793	*/
1794	# define IEM_CIMPL_DECL_TYPE_1(a_Name, a_Type0, a_Arg0) \
1795	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
1796	/**
1797	* For defining a C instruction implementation function taking one extra
1798	* argument.
1799	*
1800	* @param a_Name The name of the function
1801	* @param a_Type0 The argument type.
1802	* @param a_Arg0 The argument name.
1803	*/
1804	# define IEM_CIMPL_DEF_1(a_Name, a_Type0, a_Arg0) \
1805	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
1806	/**
1807	* Prototype version of IEM_CIMPL_DEF_1.
1808	*/
1809	# define IEM_CIMPL_PROTO_1(a_Name, a_Type0, a_Arg0) \
1810	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0))
1811	/**
1812	* For calling a C instruction implementation function taking one extra
1813	* argument.
1814	*
1815	* This special call macro adds default arguments to the call and allow us to
1816	* change these later.
1817	*
1818	* @param a_fn The name of the function.
1819	* @param a0 The name of the 1st argument.
1820	*/
1821	# define IEM_CIMPL_CALL_1(a_fn, a0) a_fn(pVCpu, cbInstr, (a0))
1822
1823	/**
1824	* For typedef'ing or declaring a C instruction implementation function taking
1825	* two extra arguments.
1826	*
1827	* @param a_Name The name of the type.
1828	* @param a_Type0 The type of the 1st argument
1829	* @param a_Arg0 The name of the 1st argument.
1830	* @param a_Type1 The type of the 2nd argument.
1831	* @param a_Arg1 The name of the 2nd argument.
1832	*/
1833	# define IEM_CIMPL_DECL_TYPE_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
1834	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
1835	/**
1836	* For defining a C instruction implementation function taking two extra
1837	* arguments.
1838	*
1839	* @param a_Name The name of the function.
1840	* @param a_Type0 The type of the 1st argument
1841	* @param a_Arg0 The name of the 1st argument.
1842	* @param a_Type1 The type of the 2nd argument.
1843	* @param a_Arg1 The name of the 2nd argument.
1844	*/
1845	# define IEM_CIMPL_DEF_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
1846	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
1847	/**
1848	* Prototype version of IEM_CIMPL_DEF_2.
1849	*/
1850	# define IEM_CIMPL_PROTO_2(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1) \
1851	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1))
1852	/**
1853	* For calling a C instruction implementation function taking two extra
1854	* arguments.
1855	*
1856	* This special call macro adds default arguments to the call and allow us to
1857	* change these later.
1858	*
1859	* @param a_fn The name of the function.
1860	* @param a0 The name of the 1st argument.
1861	* @param a1 The name of the 2nd argument.
1862	*/
1863	# define IEM_CIMPL_CALL_2(a_fn, a0, a1) a_fn(pVCpu, cbInstr, (a0), (a1))
1864
1865	/**
1866	* For typedef'ing or declaring a C instruction implementation function taking
1867	* three extra arguments.
1868	*
1869	* @param a_Name The name of the type.
1870	* @param a_Type0 The type of the 1st argument
1871	* @param a_Arg0 The name of the 1st argument.
1872	* @param a_Type1 The type of the 2nd argument.
1873	* @param a_Arg1 The name of the 2nd argument.
1874	* @param a_Type2 The type of the 3rd argument.
1875	* @param a_Arg2 The name of the 3rd argument.
1876	*/
1877	# define IEM_CIMPL_DECL_TYPE_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
1878	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
1879	/**
1880	* For defining a C instruction implementation function taking three extra
1881	* arguments.
1882	*
1883	* @param a_Name The name of the function.
1884	* @param a_Type0 The type of the 1st argument
1885	* @param a_Arg0 The name of the 1st argument.
1886	* @param a_Type1 The type of the 2nd argument.
1887	* @param a_Arg1 The name of the 2nd argument.
1888	* @param a_Type2 The type of the 3rd argument.
1889	* @param a_Arg2 The name of the 3rd argument.
1890	*/
1891	# define IEM_CIMPL_DEF_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
1892	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
1893	/**
1894	* Prototype version of IEM_CIMPL_DEF_3.
1895	*/
1896	# define IEM_CIMPL_PROTO_3(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2) \
1897	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2))
1898	/**
1899	* For calling a C instruction implementation function taking three extra
1900	* arguments.
1901	*
1902	* This special call macro adds default arguments to the call and allow us to
1903	* change these later.
1904	*
1905	* @param a_fn The name of the function.
1906	* @param a0 The name of the 1st argument.
1907	* @param a1 The name of the 2nd argument.
1908	* @param a2 The name of the 3rd argument.
1909	*/
1910	# define IEM_CIMPL_CALL_3(a_fn, a0, a1, a2) a_fn(pVCpu, cbInstr, (a0), (a1), (a2))
1911
1912
1913	/**
1914	* For typedef'ing or declaring a C instruction implementation function taking
1915	* four extra arguments.
1916	*
1917	* @param a_Name The name of the type.
1918	* @param a_Type0 The type of the 1st argument
1919	* @param a_Arg0 The name of the 1st argument.
1920	* @param a_Type1 The type of the 2nd argument.
1921	* @param a_Arg1 The name of the 2nd argument.
1922	* @param a_Type2 The type of the 3rd argument.
1923	* @param a_Arg2 The name of the 3rd argument.
1924	* @param a_Type3 The type of the 4th argument.
1925	* @param a_Arg3 The name of the 4th argument.
1926	*/
1927	# define IEM_CIMPL_DECL_TYPE_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
1928	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, a_Type3 a_Arg3))
1929	/**
1930	* For defining a C instruction implementation function taking four extra
1931	* arguments.
1932	*
1933	* @param a_Name The name of the function.
1934	* @param a_Type0 The type of the 1st argument
1935	* @param a_Arg0 The name of the 1st argument.
1936	* @param a_Type1 The type of the 2nd argument.
1937	* @param a_Arg1 The name of the 2nd argument.
1938	* @param a_Type2 The type of the 3rd argument.
1939	* @param a_Arg2 The name of the 3rd argument.
1940	* @param a_Type3 The type of the 4th argument.
1941	* @param a_Arg3 The name of the 4th argument.
1942	*/
1943	# define IEM_CIMPL_DEF_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
1944	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
1945	a_Type2 a_Arg2, a_Type3 a_Arg3))
1946	/**
1947	* Prototype version of IEM_CIMPL_DEF_4.
1948	*/
1949	# define IEM_CIMPL_PROTO_4(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3) \
1950	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
1951	a_Type2 a_Arg2, a_Type3 a_Arg3))
1952	/**
1953	* For calling a C instruction implementation function taking four extra
1954	* arguments.
1955	*
1956	* This special call macro adds default arguments to the call and allow us to
1957	* change these later.
1958	*
1959	* @param a_fn The name of the function.
1960	* @param a0 The name of the 1st argument.
1961	* @param a1 The name of the 2nd argument.
1962	* @param a2 The name of the 3rd argument.
1963	* @param a3 The name of the 4th argument.
1964	*/
1965	# define IEM_CIMPL_CALL_4(a_fn, a0, a1, a2, a3) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3))
1966
1967
1968	/**
1969	* For typedef'ing or declaring a C instruction implementation function taking
1970	* five extra arguments.
1971	*
1972	* @param a_Name The name of the type.
1973	* @param a_Type0 The type of the 1st argument
1974	* @param a_Arg0 The name of the 1st argument.
1975	* @param a_Type1 The type of the 2nd argument.
1976	* @param a_Arg1 The name of the 2nd argument.
1977	* @param a_Type2 The type of the 3rd argument.
1978	* @param a_Arg2 The name of the 3rd argument.
1979	* @param a_Type3 The type of the 4th argument.
1980	* @param a_Arg3 The name of the 4th argument.
1981	* @param a_Type4 The type of the 5th argument.
1982	* @param a_Arg4 The name of the 5th argument.
1983	*/
1984	# define IEM_CIMPL_DECL_TYPE_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
1985	IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, \
1986	a_Type0 a_Arg0, a_Type1 a_Arg1, a_Type2 a_Arg2, \
1987	a_Type3 a_Arg3, a_Type4 a_Arg4))
1988	/**
1989	* For defining a C instruction implementation function taking five extra
1990	* arguments.
1991	*
1992	* @param a_Name The name of the function.
1993	* @param a_Type0 The type of the 1st argument
1994	* @param a_Arg0 The name of the 1st argument.
1995	* @param a_Type1 The type of the 2nd argument.
1996	* @param a_Arg1 The name of the 2nd argument.
1997	* @param a_Type2 The type of the 3rd argument.
1998	* @param a_Arg2 The name of the 3rd argument.
1999	* @param a_Type3 The type of the 4th argument.
2000	* @param a_Arg3 The name of the 4th argument.
2001	* @param a_Type4 The type of the 5th argument.
2002	* @param a_Arg4 The name of the 5th argument.
2003	*/
2004	# define IEM_CIMPL_DEF_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2005	IEM_DECL_IMPL_DEF(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2006	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2007	/**
2008	* Prototype version of IEM_CIMPL_DEF_5.
2009	*/
2010	# define IEM_CIMPL_PROTO_5(a_Name, a_Type0, a_Arg0, a_Type1, a_Arg1, a_Type2, a_Arg2, a_Type3, a_Arg3, a_Type4, a_Arg4) \
2011	IEM_DECL_IMPL_PROTO(VBOXSTRICTRC, a_Name, (PVMCPUCC pVCpu, uint8_t cbInstr, a_Type0 a_Arg0, a_Type1 a_Arg1, \
2012	a_Type2 a_Arg2, a_Type3 a_Arg3, a_Type4 a_Arg4))
2013	/**
2014	* For calling a C instruction implementation function taking five extra
2015	* arguments.
2016	*
2017	* This special call macro adds default arguments to the call and allow us to
2018	* change these later.
2019	*
2020	* @param a_fn The name of the function.
2021	* @param a0 The name of the 1st argument.
2022	* @param a1 The name of the 2nd argument.
2023	* @param a2 The name of the 3rd argument.
2024	* @param a3 The name of the 4th argument.
2025	* @param a4 The name of the 5th argument.
2026	*/
2027	# define IEM_CIMPL_CALL_5(a_fn, a0, a1, a2, a3, a4) a_fn(pVCpu, cbInstr, (a0), (a1), (a2), (a3), (a4))
2028
2029	/** @} */
2030
2031
2032	/** @name Opcode Decoder Function Types.
2033	* @{ */
2034
2035	/** @typedef PFNIEMOP
2036	* Pointer to an opcode decoder function.
2037	*/
2038
2039	/** @def FNIEMOP_DEF
2040	* Define an opcode decoder function.
2041	*
2042	* We're using macors for this so that adding and removing parameters as well as
2043	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
2044	*
2045	* @param a_Name The function name.
2046	*/
2047
2048	/** @typedef PFNIEMOPRM
2049	* Pointer to an opcode decoder function with RM byte.
2050	*/
2051
2052	/** @def FNIEMOPRM_DEF
2053	* Define an opcode decoder function with RM byte.
2054	*
2055	* We're using macors for this so that adding and removing parameters as well as
2056	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
2057	*
2058	* @param a_Name The function name.
2059	*/
2060
2061	#if defined(__GNUC__) && defined(RT_ARCH_X86)
2062	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
2063	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2064	# define FNIEMOP_DEF(a_Name) \
2065	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
2066	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2067	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2068	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2069	IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2070
2071	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
2072	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
2073	typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2074	# define FNIEMOP_DEF(a_Name) \
2075	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2076	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2077	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2078	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2079	IEM_STATIC /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2080
2081	#elif defined(__GNUC__)
2082	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2083	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2084	# define FNIEMOP_DEF(a_Name) \
2085	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
2086	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2087	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
2088	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2089	IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
2090
2091	#else
2092	typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
2093	typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
2094	# define FNIEMOP_DEF(a_Name) \
2095	IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
2096	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2097	IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
2098	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
2099	IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
2100
2101	#endif
2102	#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
2103
2104	/**
2105	* Call an opcode decoder function.
2106	*
2107	* We're using macors for this so that adding and removing parameters can be
2108	* done as we please. See FNIEMOP_DEF.
2109	*/
2110	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
2111
2112	/**
2113	* Call a common opcode decoder function taking one extra argument.
2114	*
2115	* We're using macors for this so that adding and removing parameters can be
2116	* done as we please. See FNIEMOP_DEF_1.
2117	*/
2118	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
2119
2120	/**
2121	* Call a common opcode decoder function taking one extra argument.
2122	*
2123	* We're using macors for this so that adding and removing parameters can be
2124	* done as we please. See FNIEMOP_DEF_1.
2125	*/
2126	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
2127	/** @} */
2128
2129
2130	/** @name Misc Helpers
2131	* @{ */
2132
2133	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
2134	* due to GCC lacking knowledge about the value range of a switch. */
2135	#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
2136
2137	/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
2138	#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
2139
2140	/**
2141	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2142	* occation.
2143	*/
2144	#ifdef LOG_ENABLED
2145	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2146	do { \
2147	/Log/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
2148	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2149	} while (0)
2150	#else
2151	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
2152	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2153	#endif
2154
2155	/**
2156	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
2157	* occation using the supplied logger statement.
2158	*
2159	* @param a_LoggerArgs What to log on failure.
2160	*/
2161	#ifdef LOG_ENABLED
2162	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2163	do { \
2164	LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
2165	/LogFunc(a_LoggerArgs);/ \
2166	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
2167	} while (0)
2168	#else
2169	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
2170	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
2171	#endif
2172
2173	/**
2174	* Check if we're currently executing in real or virtual 8086 mode.
2175	*
2176	* @returns @c true if it is, @c false if not.
2177	* @param a_pVCpu The IEM state of the current CPU.
2178	*/
2179	#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2180
2181	/**
2182	* Check if we're currently executing in virtual 8086 mode.
2183	*
2184	* @returns @c true if it is, @c false if not.
2185	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2186	*/
2187	#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
2188
2189	/**
2190	* Check if we're currently executing in long mode.
2191	*
2192	* @returns @c true if it is, @c false if not.
2193	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2194	*/
2195	#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
2196
2197	/**
2198	* Check if we're currently executing in a 64-bit code segment.
2199	*
2200	* @returns @c true if it is, @c false if not.
2201	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2202	*/
2203	#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
2204
2205	/**
2206	* Check if we're currently executing in real mode.
2207	*
2208	* @returns @c true if it is, @c false if not.
2209	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2210	*/
2211	#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
2212
2213	/**
2214	* Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
2215	* @returns PCCPUMFEATURES
2216	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2217	*/
2218	#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
2219
2220	/**
2221	* Returns a (const) pointer to the CPUMFEATURES for the host CPU.
2222	* @returns PCCPUMFEATURES
2223	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2224	*/
2225	#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.HostFeatures))
2226
2227	/**
2228	* Evaluates to true if we're presenting an Intel CPU to the guest.
2229	*/
2230	#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
2231
2232	/**
2233	* Evaluates to true if we're presenting an AMD CPU to the guest.
2234	*/
2235	#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD \|\| (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
2236
2237	/**
2238	* Check if the address is canonical.
2239	*/
2240	#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
2241
2242	/**
2243	* Gets the effective VEX.VVVV value.
2244	*
2245	* The 4th bit is ignored if not 64-bit code.
2246	* @returns effective V-register value.
2247	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
2248	*/
2249	#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
2250	((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
2251
2252
2253	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2254
2255	/**
2256	* Check if the guest has entered VMX root operation.
2257	*/
2258	# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
2259
2260	/**
2261	* Check if the guest has entered VMX non-root operation.
2262	*/
2263	# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
2264
2265	/**
2266	* Check if the nested-guest has the given Pin-based VM-execution control set.
2267	*/
2268	# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
2269	(CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
2270
2271	/**
2272	* Check if the nested-guest has the given Processor-based VM-execution control set.
2273	*/
2274	# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
2275	(CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
2276
2277	/**
2278	* Check if the nested-guest has the given Secondary Processor-based VM-execution
2279	* control set.
2280	*/
2281	# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
2282	(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
2283
2284	/** Gets the guest-physical address of the shadows VMCS for the given VCPU. */
2285	# define IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs)
2286
2287	/** Whether a shadow VMCS is present for the given VCPU. */
2288	# define IEM_VMX_HAS_SHADOW_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_SHADOW_VMCS(a_pVCpu) != NIL_RTGCPHYS)
2289
2290	/** Gets the VMXON region pointer. */
2291	# define IEM_VMX_GET_VMXON_PTR(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
2292
2293	/** Gets the guest-physical address of the current VMCS for the given VCPU. */
2294	# define IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) ((a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs)
2295
2296	/** Whether a current VMCS is present for the given VCPU. */
2297	# define IEM_VMX_HAS_CURRENT_VMCS(a_pVCpu) RT_BOOL(IEM_VMX_GET_CURRENT_VMCS(a_pVCpu) != NIL_RTGCPHYS)
2298
2299	/** Assigns the guest-physical address of the current VMCS for the given VCPU. */
2300	# define IEM_VMX_SET_CURRENT_VMCS(a_pVCpu, a_GCPhysVmcs) \
2301	do \
2302	{ \
2303	Assert((a_GCPhysVmcs) != NIL_RTGCPHYS); \
2304	(a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = (a_GCPhysVmcs); \
2305	} while (0)
2306
2307	/** Clears any current VMCS for the given VCPU. */
2308	# define IEM_VMX_CLEAR_CURRENT_VMCS(a_pVCpu) \
2309	do \
2310	{ \
2311	(a_pVCpu)->cpum.GstCtx.hwvirt.vmx.GCPhysVmcs = NIL_RTGCPHYS; \
2312	} while (0)
2313
2314	/**
2315	* Invokes the VMX VM-exit handler for an instruction intercept.
2316	*/
2317	# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
2318	do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
2319
2320	/**
2321	* Invokes the VMX VM-exit handler for an instruction intercept where the
2322	* instruction provides additional VM-exit information.
2323	*/
2324	# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
2325	do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
2326
2327	/**
2328	* Invokes the VMX VM-exit handler for a task switch.
2329	*/
2330	# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
2331	do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
2332
2333	/**
2334	* Invokes the VMX VM-exit handler for MWAIT.
2335	*/
2336	# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
2337	do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
2338
2339	/**
2340	* Invokes the VMX VM-exit handler for EPT faults.
2341	*/
2342	# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) \
2343	do { return iemVmxVmexitEpt(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr); } while (0)
2344
2345	/**
2346	* Invokes the VMX VM-exit handler.
2347	*/
2348	# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
2349	do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
2350
2351	#else
2352	# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (false)
2353	# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (false)
2354	# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr) (false)
2355	# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr) (false)
2356	# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr) (false)
2357	# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2358	# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2359	# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2360	# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2361	# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
2362	# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) do { return VERR_VMX_IPE_1; } while (0)
2363
2364	#endif
2365
2366	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2367	/**
2368	* Check if an SVM control/instruction intercept is set.
2369	*/
2370	# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
2371	(CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
2372
2373	/**
2374	* Check if an SVM read CRx intercept is set.
2375	*/
2376	# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
2377	(CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
2378
2379	/**
2380	* Check if an SVM write CRx intercept is set.
2381	*/
2382	# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
2383	(CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
2384
2385	/**
2386	* Check if an SVM read DRx intercept is set.
2387	*/
2388	# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
2389	(CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
2390
2391	/**
2392	* Check if an SVM write DRx intercept is set.
2393	*/
2394	# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
2395	(CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
2396
2397	/**
2398	* Check if an SVM exception intercept is set.
2399	*/
2400	# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
2401	(CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
2402
2403	/**
2404	* Invokes the SVM \#VMEXIT handler for the nested-guest.
2405	*/
2406	# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
2407	do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
2408
2409	/**
2410	* Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
2411	* corresponding decode assist information.
2412	*/
2413	# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
2414	do \
2415	{ \
2416	uint64_t uExitInfo1; \
2417	if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
2418	&& (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
2419	uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK \| ((a_iGReg) & 7); \
2420	else \
2421	uExitInfo1 = 0; \
2422	IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
2423	} while (0)
2424
2425	/** Check and handles SVM nested-guest instruction intercept and updates
2426	* NRIP if needed.
2427	*/
2428	# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
2429	do \
2430	{ \
2431	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
2432	{ \
2433	IEM_SVM_UPDATE_NRIP(a_pVCpu); \
2434	IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
2435	} \
2436	} while (0)
2437
2438	/** Checks and handles SVM nested-guest CR0 read intercept. */
2439	# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
2440	do \
2441	{ \
2442	if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
2443	{ /* probably likely */ } \
2444	else \
2445	{ \
2446	IEM_SVM_UPDATE_NRIP(a_pVCpu); \
2447	IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
2448	} \
2449	} while (0)
2450
2451	/**
2452	* Updates the NextRIP (NRI) field in the nested-guest VMCB.
2453	*/
2454	# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
2455	do { \
2456	if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
2457	CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
2458	} while (0)
2459
2460	#else
2461	# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
2462	# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
2463	# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
2464	# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
2465	# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
2466	# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
2467	# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
2468	# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
2469	# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { } while (0)
2470	# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) do { } while (0)
2471	# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
2472
2473	#endif
2474
2475	/** @} */
2476
2477
2478
2479	/**
2480	* Selector descriptor table entry as fetched by iemMemFetchSelDesc.
2481	*/
2482	typedef union IEMSELDESC
2483	{
2484	/** The legacy view. */
2485	X86DESC Legacy;
2486	/** The long mode view. */
2487	X86DESC64 Long;
2488	} IEMSELDESC;
2489	/** Pointer to a selector descriptor table entry. */
2490	typedef IEMSELDESC *PIEMSELDESC;
2491
2492	/** @name Raising Exceptions.
2493	* @{ */
2494	VBOXSTRICTRC iemTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, uint32_t uNextEip, uint32_t fFlags,
2495	uint16_t uErr, uint64_t uCr2, RTSEL SelTSS, PIEMSELDESC pNewDescTSS) RT_NOEXCEPT;
2496
2497	VBOXSTRICTRC iemRaiseXcptOrInt(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector, uint32_t fFlags,
2498	uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
2499	#ifdef IEM_WITH_SETJMP
2500	DECL_NO_RETURN(void) iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t u8Vector,
2501	uint32_t fFlags, uint16_t uErr, uint64_t uCr2) RT_NOEXCEPT;
2502	#endif
2503	VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT;
2504	VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT;
2505	VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT;
2506	VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT;
2507	VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT;
2508	VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2509	VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT;
2510	VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
2511	VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2512	/VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;/
2513	VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2514	VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2515	VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2516	VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2517	VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT;
2518	VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT;
2519	#ifdef IEM_WITH_SETJMP
2520	DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2521	#endif
2522	VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2523	VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT;
2524	VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2525	#ifdef IEM_WITH_SETJMP
2526	DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2527	#endif
2528	VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2529	#ifdef IEM_WITH_SETJMP
2530	DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT;
2531	#endif
2532	VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2533	#ifdef IEM_WITH_SETJMP
2534	DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT;
2535	#endif
2536	VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
2537	#ifdef IEM_WITH_SETJMP
2538	DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT;
2539	#endif
2540	VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu);
2541	VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
2542
2543	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError);
2544	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix);
2545	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode);
2546
2547	/**
2548	* Macro for calling iemCImplRaiseDivideError().
2549	*
2550	* This enables us to add/remove arguments and force different levels of
2551	* inlining as we wish.
2552	*
2553	* @return Strict VBox status code.
2554	*/
2555	#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
2556
2557	/**
2558	* Macro for calling iemCImplRaiseInvalidLockPrefix().
2559	*
2560	* This enables us to add/remove arguments and force different levels of
2561	* inlining as we wish.
2562	*
2563	* @return Strict VBox status code.
2564	*/
2565	#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
2566
2567	/**
2568	* Macro for calling iemCImplRaiseInvalidOpcode().
2569	*
2570	* This enables us to add/remove arguments and force different levels of
2571	* inlining as we wish.
2572	*
2573	* @return Strict VBox status code.
2574	*/
2575	#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
2576	/** @} */
2577
2578	/** @name Register Access.
2579	* @{ */
2580	VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT;
2581	VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT;
2582	VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT;
2583	VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT;
2584	/** @} */
2585
2586	/** @name FPU access and helpers.
2587	* @{ */
2588	void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT;
2589	void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2590	void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT;
2591	void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
2592	void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT;
2593	void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
2594	uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2595	void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
2596	uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2597	void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2598	void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
2599	void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
2600	void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2601	void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT;
2602	void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2603	void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
2604	void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2605	void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT;
2606	void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2607	void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT;
2608	void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
2609	void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT;
2610	void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT;
2611	void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT;
2612	/** @} */
2613
2614	/** @name Memory access.
2615	* @{ */
2616	VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess) RT_NOEXCEPT;
2617	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
2618	#ifndef IN_RING3
2619	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT;
2620	#endif
2621	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT;
2622	VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT;
2623	VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT;
2624	VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT;
2625
2626	#ifdef IEM_WITH_CODE_TLB
2627	void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT;
2628	#else
2629	VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT;
2630	#endif
2631	#ifdef IEM_WITH_SETJMP
2632	uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2633	uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2634	uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2635	uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT;
2636	#else
2637	VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT;
2638	VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
2639	VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
2640	VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
2641	VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT;
2642	VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
2643	VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
2644	VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT;
2645	VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
2646	VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
2647	VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT;
2648	#endif
2649
2650	VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2651	VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2652	VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2653	VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2654	VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2655	VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2656	VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2657	VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2658	VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2659	VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2660	VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2661	VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2662	VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
2663	RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT;
2664	#ifdef IEM_WITH_SETJMP
2665	uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2666	uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2667	uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2668	uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2669	uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2670	void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2671	void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2672	void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2673	void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2674	void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2675	void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2676	#endif
2677
2678	VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2679	VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2680	VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2681	VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2682	VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT;
2683
2684	VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
2685	VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
2686	VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
2687	VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
2688	VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
2689	VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
2690	VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
2691	VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
2692	VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT;
2693	#ifdef IEM_WITH_SETJMP
2694	void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT;
2695	void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT;
2696	void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT;
2697	void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT;
2698	void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
2699	void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT;
2700	void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
2701	void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT;
2702	#endif
2703
2704	VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT;
2705	VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT;
2706	VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT;
2707	VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
2708	VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT;
2709	VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
2710	VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
2711	VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
2712	VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT;
2713	VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT;
2714	VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT;
2715	VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT;
2716	VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT;
2717	VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT;
2718	VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT;
2719	VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
2720	VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
2721	VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT;
2722	/** @} */
2723
2724	/** @name IEMAllCImpl.cpp
2725	* @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/'
2726	* @{ */
2727	IEM_CIMPL_PROTO_0(iemCImpl_popa_16);
2728	IEM_CIMPL_PROTO_0(iemCImpl_popa_32);
2729	IEM_CIMPL_PROTO_0(iemCImpl_pusha_16);
2730	IEM_CIMPL_PROTO_0(iemCImpl_pusha_32);
2731	IEM_CIMPL_PROTO_1(iemCImpl_pushf, IEMMODE, enmEffOpSize);
2732	IEM_CIMPL_PROTO_1(iemCImpl_popf, IEMMODE, enmEffOpSize);
2733	IEM_CIMPL_PROTO_1(iemCImpl_call_16, uint16_t, uNewPC);
2734	IEM_CIMPL_PROTO_1(iemCImpl_call_rel_16, int16_t, offDisp);
2735	IEM_CIMPL_PROTO_1(iemCImpl_call_32, uint32_t, uNewPC);
2736	IEM_CIMPL_PROTO_1(iemCImpl_call_rel_32, int32_t, offDisp);
2737	IEM_CIMPL_PROTO_1(iemCImpl_call_64, uint64_t, uNewPC);
2738	IEM_CIMPL_PROTO_1(iemCImpl_call_rel_64, int64_t, offDisp);
2739	IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskSegment, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
2740	IEM_CIMPL_PROTO_4(iemCImpl_BranchTaskGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
2741	IEM_CIMPL_PROTO_4(iemCImpl_BranchCallGate, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
2742	IEM_CIMPL_PROTO_4(iemCImpl_BranchSysSel, uint16_t, uSel, IEMBRANCH, enmBranch, IEMMODE, enmEffOpSize, PIEMSELDESC, pDesc);
2743	IEM_CIMPL_PROTO_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
2744	IEM_CIMPL_PROTO_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize);
2745	IEM_CIMPL_PROTO_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop);
2746	IEM_CIMPL_PROTO_2(iemCImpl_retn, IEMMODE, enmEffOpSize, uint16_t, cbPop);
2747	IEM_CIMPL_PROTO_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters);
2748	IEM_CIMPL_PROTO_1(iemCImpl_leave, IEMMODE, enmEffOpSize);
2749	IEM_CIMPL_PROTO_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt);
2750	IEM_CIMPL_PROTO_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize);
2751	IEM_CIMPL_PROTO_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp);
2752	IEM_CIMPL_PROTO_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize);
2753	IEM_CIMPL_PROTO_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize);
2754	IEM_CIMPL_PROTO_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize);
2755	IEM_CIMPL_PROTO_1(iemCImpl_iret, IEMMODE, enmEffOpSize);
2756	IEM_CIMPL_PROTO_0(iemCImpl_loadall286);
2757	IEM_CIMPL_PROTO_0(iemCImpl_syscall);
2758	IEM_CIMPL_PROTO_0(iemCImpl_sysret);
2759	IEM_CIMPL_PROTO_0(iemCImpl_sysenter);
2760	IEM_CIMPL_PROTO_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize);
2761	IEM_CIMPL_PROTO_2(iemCImpl_LoadSReg, uint8_t, iSegReg, uint16_t, uSel);
2762	IEM_CIMPL_PROTO_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel);
2763	IEM_CIMPL_PROTO_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize);
2764	IEM_CIMPL_PROTO_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize);
2765	IEM_CIMPL_PROTO_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite);
2766	IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar);
2767	IEM_CIMPL_PROTO_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar);
2768	IEM_CIMPL_PROTO_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
2769	IEM_CIMPL_PROTO_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2770	IEM_CIMPL_PROTO_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize);
2771	IEM_CIMPL_PROTO_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2772	IEM_CIMPL_PROTO_1(iemCImpl_lldt, uint16_t, uNewLdt);
2773	IEM_CIMPL_PROTO_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
2774	IEM_CIMPL_PROTO_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2775	IEM_CIMPL_PROTO_1(iemCImpl_ltr, uint16_t, uNewTr);
2776	IEM_CIMPL_PROTO_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
2777	IEM_CIMPL_PROTO_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2778	IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg);
2779	IEM_CIMPL_PROTO_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize);
2780	IEM_CIMPL_PROTO_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2781	IEM_CIMPL_PROTO_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg);
2782	IEM_CIMPL_PROTO_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg);
2783	IEM_CIMPL_PROTO_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst);
2784	IEM_CIMPL_PROTO_0(iemCImpl_clts);
2785	IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg);
2786	IEM_CIMPL_PROTO_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg);
2787	IEM_CIMPL_PROTO_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg);
2788	IEM_CIMPL_PROTO_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg);
2789	IEM_CIMPL_PROTO_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage);
2790	IEM_CIMPL_PROTO_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType);
2791	IEM_CIMPL_PROTO_0(iemCImpl_invd);
2792	IEM_CIMPL_PROTO_0(iemCImpl_wbinvd);
2793	IEM_CIMPL_PROTO_0(iemCImpl_rsm);
2794	IEM_CIMPL_PROTO_0(iemCImpl_rdtsc);
2795	IEM_CIMPL_PROTO_0(iemCImpl_rdtscp);
2796	IEM_CIMPL_PROTO_0(iemCImpl_rdpmc);
2797	IEM_CIMPL_PROTO_0(iemCImpl_rdmsr);
2798	IEM_CIMPL_PROTO_0(iemCImpl_wrmsr);
2799	IEM_CIMPL_PROTO_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
2800	IEM_CIMPL_PROTO_1(iemCImpl_in_eAX_DX, uint8_t, cbReg);
2801	IEM_CIMPL_PROTO_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg);
2802	IEM_CIMPL_PROTO_1(iemCImpl_out_DX_eAX, uint8_t, cbReg);
2803	IEM_CIMPL_PROTO_0(iemCImpl_cli);
2804	IEM_CIMPL_PROTO_0(iemCImpl_sti);
2805	IEM_CIMPL_PROTO_0(iemCImpl_hlt);
2806	IEM_CIMPL_PROTO_1(iemCImpl_monitor, uint8_t, iEffSeg);
2807	IEM_CIMPL_PROTO_0(iemCImpl_mwait);
2808	IEM_CIMPL_PROTO_0(iemCImpl_swapgs);
2809	IEM_CIMPL_PROTO_0(iemCImpl_cpuid);
2810	IEM_CIMPL_PROTO_1(iemCImpl_aad, uint8_t, bImm);
2811	IEM_CIMPL_PROTO_1(iemCImpl_aam, uint8_t, bImm);
2812	IEM_CIMPL_PROTO_0(iemCImpl_daa);
2813	IEM_CIMPL_PROTO_0(iemCImpl_das);
2814	IEM_CIMPL_PROTO_0(iemCImpl_aaa);
2815	IEM_CIMPL_PROTO_0(iemCImpl_aas);
2816	IEM_CIMPL_PROTO_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound);
2817	IEM_CIMPL_PROTO_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound);
2818	IEM_CIMPL_PROTO_0(iemCImpl_xgetbv);
2819	IEM_CIMPL_PROTO_0(iemCImpl_xsetbv);
2820	IEM_CIMPL_PROTO_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
2821	PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags);
2822	IEM_CIMPL_PROTO_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
2823	IEM_CIMPL_PROTO_1(iemCImpl_finit, bool, fCheckXcpts);
2824	IEM_CIMPL_PROTO_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
2825	IEM_CIMPL_PROTO_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
2826	IEM_CIMPL_PROTO_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
2827	IEM_CIMPL_PROTO_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize);
2828	IEM_CIMPL_PROTO_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
2829	IEM_CIMPL_PROTO_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
2830	IEM_CIMPL_PROTO_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff);
2831	IEM_CIMPL_PROTO_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2832	IEM_CIMPL_PROTO_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst);
2833	IEM_CIMPL_PROTO_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
2834	IEM_CIMPL_PROTO_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc);
2835	IEM_CIMPL_PROTO_1(iemCImpl_fldcw, uint16_t, u16Fcw);
2836	IEM_CIMPL_PROTO_1(iemCImpl_fxch_underflow, uint8_t, iStReg);
2837	IEM_CIMPL_PROTO_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop);
2838	/** @} */
2839
2840	/** @name IEMAllCImplStrInstr.cpp.h
2841	* @note sed -e '/IEM_CIMPL_DEF_/!d' -e 's/IEM_CIMPL_DEF_/IEM_CIMPL_PROTO_/' -e 's/$/;/' -e 's/RT_CONCAT4(//' \
2842	* -e 's/,ADDR_SIZE)/64/g' -e 's/,OP_SIZE,/64/g' -e 's/,OP_rAX,/rax/g' IEMAllCImplStrInstr.cpp.h
2843	* @{ */
2844	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr16, uint8_t, iEffSeg);
2845	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr16, uint8_t, iEffSeg);
2846	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m16);
2847	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m16);
2848	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr16, uint8_t, iEffSeg);
2849	IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m16);
2850	IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m16, int8_t, iEffSeg);
2851	IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr16, bool, fIoChecked);
2852	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr16, bool, fIoChecked);
2853	IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
2854	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr16, uint8_t, iEffSeg, bool, fIoChecked);
2855
2856	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr16, uint8_t, iEffSeg);
2857	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr16, uint8_t, iEffSeg);
2858	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m16);
2859	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m16);
2860	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr16, uint8_t, iEffSeg);
2861	IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m16);
2862	IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m16, int8_t, iEffSeg);
2863	IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr16, bool, fIoChecked);
2864	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr16, bool, fIoChecked);
2865	IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
2866	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr16, uint8_t, iEffSeg, bool, fIoChecked);
2867
2868	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr16, uint8_t, iEffSeg);
2869	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr16, uint8_t, iEffSeg);
2870	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m16);
2871	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m16);
2872	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr16, uint8_t, iEffSeg);
2873	IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m16);
2874	IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m16, int8_t, iEffSeg);
2875	IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr16, bool, fIoChecked);
2876	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr16, bool, fIoChecked);
2877	IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
2878	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr16, uint8_t, iEffSeg, bool, fIoChecked);
2879
2880
2881	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr32, uint8_t, iEffSeg);
2882	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr32, uint8_t, iEffSeg);
2883	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m32);
2884	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m32);
2885	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr32, uint8_t, iEffSeg);
2886	IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m32);
2887	IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m32, int8_t, iEffSeg);
2888	IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr32, bool, fIoChecked);
2889	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr32, bool, fIoChecked);
2890	IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2891	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2892
2893	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr32, uint8_t, iEffSeg);
2894	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr32, uint8_t, iEffSeg);
2895	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m32);
2896	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m32);
2897	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr32, uint8_t, iEffSeg);
2898	IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m32);
2899	IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m32, int8_t, iEffSeg);
2900	IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr32, bool, fIoChecked);
2901	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr32, bool, fIoChecked);
2902	IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2903	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2904
2905	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr32, uint8_t, iEffSeg);
2906	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr32, uint8_t, iEffSeg);
2907	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m32);
2908	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m32);
2909	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr32, uint8_t, iEffSeg);
2910	IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m32);
2911	IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m32, int8_t, iEffSeg);
2912	IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr32, bool, fIoChecked);
2913	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr32, bool, fIoChecked);
2914	IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2915	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2916
2917	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr32, uint8_t, iEffSeg);
2918	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr32, uint8_t, iEffSeg);
2919	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m32);
2920	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m32);
2921	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr32, uint8_t, iEffSeg);
2922	IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m32);
2923	IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m32, int8_t, iEffSeg);
2924	IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr32, bool, fIoChecked);
2925	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr32, bool, fIoChecked);
2926	IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2927	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr32, uint8_t, iEffSeg, bool, fIoChecked);
2928
2929
2930	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op8_addr64, uint8_t, iEffSeg);
2931	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op8_addr64, uint8_t, iEffSeg);
2932	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_al_m64);
2933	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_al_m64);
2934	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op8_addr64, uint8_t, iEffSeg);
2935	IEM_CIMPL_PROTO_0(iemCImpl_stos_al_m64);
2936	IEM_CIMPL_PROTO_1(iemCImpl_lods_al_m64, int8_t, iEffSeg);
2937	IEM_CIMPL_PROTO_1(iemCImpl_ins_op8_addr64, bool, fIoChecked);
2938	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op8_addr64, bool, fIoChecked);
2939	IEM_CIMPL_PROTO_2(iemCImpl_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2940	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op8_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2941
2942	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op16_addr64, uint8_t, iEffSeg);
2943	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op16_addr64, uint8_t, iEffSeg);
2944	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_ax_m64);
2945	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_ax_m64);
2946	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op16_addr64, uint8_t, iEffSeg);
2947	IEM_CIMPL_PROTO_0(iemCImpl_stos_ax_m64);
2948	IEM_CIMPL_PROTO_1(iemCImpl_lods_ax_m64, int8_t, iEffSeg);
2949	IEM_CIMPL_PROTO_1(iemCImpl_ins_op16_addr64, bool, fIoChecked);
2950	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op16_addr64, bool, fIoChecked);
2951	IEM_CIMPL_PROTO_2(iemCImpl_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2952	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op16_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2953
2954	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op32_addr64, uint8_t, iEffSeg);
2955	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op32_addr64, uint8_t, iEffSeg);
2956	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_eax_m64);
2957	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_eax_m64);
2958	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op32_addr64, uint8_t, iEffSeg);
2959	IEM_CIMPL_PROTO_0(iemCImpl_stos_eax_m64);
2960	IEM_CIMPL_PROTO_1(iemCImpl_lods_eax_m64, int8_t, iEffSeg);
2961	IEM_CIMPL_PROTO_1(iemCImpl_ins_op32_addr64, bool, fIoChecked);
2962	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op32_addr64, bool, fIoChecked);
2963	IEM_CIMPL_PROTO_2(iemCImpl_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2964	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op32_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2965
2966	IEM_CIMPL_PROTO_1(iemCImpl_repe_cmps_op64_addr64, uint8_t, iEffSeg);
2967	IEM_CIMPL_PROTO_1(iemCImpl_repne_cmps_op64_addr64, uint8_t, iEffSeg);
2968	IEM_CIMPL_PROTO_0(iemCImpl_repe_scas_rax_m64);
2969	IEM_CIMPL_PROTO_0(iemCImpl_repne_scas_rax_m64);
2970	IEM_CIMPL_PROTO_1(iemCImpl_rep_movs_op64_addr64, uint8_t, iEffSeg);
2971	IEM_CIMPL_PROTO_0(iemCImpl_stos_rax_m64);
2972	IEM_CIMPL_PROTO_1(iemCImpl_lods_rax_m64, int8_t, iEffSeg);
2973	IEM_CIMPL_PROTO_1(iemCImpl_ins_op64_addr64, bool, fIoChecked);
2974	IEM_CIMPL_PROTO_1(iemCImpl_rep_ins_op64_addr64, bool, fIoChecked);
2975	IEM_CIMPL_PROTO_2(iemCImpl_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2976	IEM_CIMPL_PROTO_2(iemCImpl_rep_outs_op64_addr64, uint8_t, iEffSeg, bool, fIoChecked);
2977	/** @} */
2978
2979	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2980	VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) RT_NOEXCEPT;
2981	VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) RT_NOEXCEPT;
2982	VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr) RT_NOEXCEPT;
2983	VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr) RT_NOEXCEPT;
2984	VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr) RT_NOEXCEPT;
2985	VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu) RT_NOEXCEPT;
2986	VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr) RT_NOEXCEPT;
2987	VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT;
2988	VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr) RT_NOEXCEPT;
2989	VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port,
2990	bool fImm, uint8_t cbAccess, uint8_t cbInstr) RT_NOEXCEPT;
2991	VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess,
2992	bool fRep, VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr) RT_NOEXCEPT;
2993	VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
2994	VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
2995	VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
2996	VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
2997	VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
2998	VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT;
2999	VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT;
3000	VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw,
3001	RTGCPTR GCPtrEffDst, uint8_t cbInstr) RT_NOEXCEPT;
3002	VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr) RT_NOEXCEPT;
3003	VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu) RT_NOEXCEPT;
3004	VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT;
3005	uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT;
3006	void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg) RT_NOEXCEPT;
3007	VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc,
3008	uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT;
3009	bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr) RT_NOEXCEPT;
3010	IEM_CIMPL_PROTO_0(iemCImpl_vmxoff);
3011	IEM_CIMPL_PROTO_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon);
3012	IEM_CIMPL_PROTO_0(iemCImpl_vmlaunch);
3013	IEM_CIMPL_PROTO_0(iemCImpl_vmresume);
3014	IEM_CIMPL_PROTO_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3015	IEM_CIMPL_PROTO_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3016	IEM_CIMPL_PROTO_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs);
3017	IEM_CIMPL_PROTO_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField);
3018	IEM_CIMPL_PROTO_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField);
3019	IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField);
3020	IEM_CIMPL_PROTO_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField);
3021	IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField);
3022	IEM_CIMPL_PROTO_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField);
3023	IEM_CIMPL_PROTO_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType);
3024	IEM_CIMPL_PROTO_3(iemCImpl_invept, uint8_t, iEffSeg, RTGCPTR, GCPtrInveptDesc, uint64_t, uInveptType);
3025	IEM_CIMPL_PROTO_0(iemCImpl_vmx_pause);
3026	IEM_CIMPL_PROTO_0(iemCImpl_vmcall);
3027	#endif
3028
3029	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3030	VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) RT_NOEXCEPT;
3031	VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2) RT_NOEXCEPT;
3032	VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
3033	uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr) RT_NOEXCEPT;
3034	VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite) RT_NOEXCEPT;
3035	IEM_CIMPL_PROTO_0(iemCImpl_vmrun);
3036	IEM_CIMPL_PROTO_0(iemCImpl_vmload);
3037	IEM_CIMPL_PROTO_0(iemCImpl_vmsave);
3038	IEM_CIMPL_PROTO_0(iemCImpl_clgi);
3039	IEM_CIMPL_PROTO_0(iemCImpl_stgi);
3040	IEM_CIMPL_PROTO_0(iemCImpl_invlpga);
3041	IEM_CIMPL_PROTO_0(iemCImpl_skinit);
3042	IEM_CIMPL_PROTO_0(iemCImpl_svm_pause);
3043	IEM_CIMPL_PROTO_0(iemCImpl_vmmcall);
3044	#endif
3045
3046	IEM_CIMPL_PROTO_1(iemCImpl_Hypercall, uint16_t, uDisOpcode);
3047
3048
3049	extern const PFNIEMOP g_apfnOneByteMap[256];
3050
3051	/** @} */
3052
3053	RT_C_DECLS_END
3054
3055	#endif /* !VMM_INCLUDED_SRC_include_IEMInternal_h */
3056

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInternal.h@ 94791

Download in other formats: