IEMAll.cpp@ 52077

Last change on this file since 52077 was 52077, checked in by vboxsync, 10 years ago
VMM/IEM: Block further NMIs while injecting an NMI.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 418.9 KB

Line
1	/* $Id: IEMAll.cpp 52077 2014-07-17 13:03:19Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_iem IEM - Interpreted Execution Manager
20	*
21	* The interpreted exeuction manager (IEM) is for executing short guest code
22	* sequences that are causing too many exits / virtualization traps. It will
23	* also be used to interpret single instructions, thus replacing the selective
24	* interpreters in EM and IOM.
25	*
26	* Design goals:
27	* - Relatively small footprint, although we favour speed and correctness
28	* over size.
29	* - Reasonably fast.
30	* - Correctly handle lock prefixed instructions.
31	* - Complete instruction set - eventually.
32	* - Refactorable into a recompiler, maybe.
33	* - Replace EMInterpret*.
34	*
35	* Using the existing disassembler has been considered, however this is thought
36	* to conflict with speed as the disassembler chews things a bit too much while
37	* leaving us with a somewhat complicated state to interpret afterwards.
38	*
39	*
40	* The current code is very much work in progress. You've been warned!
41	*
42	*
43	* @section sec_iem_fpu_instr FPU Instructions
44	*
45	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46	* same or equivalent instructions on the host FPU. To make life easy, we also
47	* let the FPU prioritize the unmasked exceptions for us. This however, only
48	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
50	* can trigger spurious FPU exceptions.
51	*
52	* The guest FPU state is not loaded into the host CPU and kept there till we
53	* leave IEM because the calling conventions have declared an all year open
54	* season on much of the FPU state. For instance an innocent looking call to
55	* memcpy might end up using a whole bunch of XMM or MM registers if the
56	* particular implementation finds it worthwhile.
57	*
58	*
59	* @section sec_iem_logging Logging
60	*
61	* The IEM code uses the \"IEM\" log group for the main logging. The different
62	* logging levels/flags are generally used for the following purposes:
63	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64	* - Flow (LogFlow): Basic enter/exit IEM state info.
65	* - Level 2 (Log2): ?
66	* - Level 3 (Log3): More detailed enter/exit IEM state info.
67	* - Level 4 (Log4): Decoding mnemonics w/ EIP.
68	* - Level 5 (Log5): Decoding details.
69	* - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
70	*
71	*/
72
73	/** @def IEM_VERIFICATION_MODE_MINIMAL
74	* Use for pitting IEM against EM or something else in ring-0 or raw-mode
75	* context. */
76	//#define IEM_VERIFICATION_MODE_MINIMAL
77	//#define IEM_LOG_MEMORY_WRITES
78	//#define IEM_IMPLEMENTS_TASKSWITCH
79
80	/*******************************************************************************
81	* Header Files *
82	*******************************************************************************/
83	#define LOG_GROUP LOG_GROUP_IEM
84	#include <VBox/vmm/iem.h>
85	#include <VBox/vmm/cpum.h>
86	#include <VBox/vmm/pdm.h>
87	#include <VBox/vmm/pgm.h>
88	#include <internal/pgm.h>
89	#include <VBox/vmm/iom.h>
90	#include <VBox/vmm/em.h>
91	#include <VBox/vmm/hm.h>
92	#include <VBox/vmm/tm.h>
93	#include <VBox/vmm/dbgf.h>
94	#include <VBox/vmm/dbgftrace.h>
95	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
96	# include <VBox/vmm/patm.h>
97	# if defined(VBOX_WITH_CALL_RECORD) \|\| defined(REM_MONITOR_CODE_PAGES)
98	# include <VBox/vmm/csam.h>
99	# endif
100	#endif
101	#include "IEMInternal.h"
102	#ifdef IEM_VERIFICATION_MODE_FULL
103	# include <VBox/vmm/rem.h>
104	# include <VBox/vmm/mm.h>
105	#endif
106	#include <VBox/vmm/vm.h>
107	#include <VBox/log.h>
108	#include <VBox/err.h>
109	#include <VBox/param.h>
110	#include <VBox/dis.h>
111	#include <VBox/disopcode.h>
112	#include <iprt/assert.h>
113	#include <iprt/string.h>
114	#include <iprt/x86.h>
115
116
117
118	/*******************************************************************************
119	* Structures and Typedefs *
120	*******************************************************************************/
121	/** @typedef PFNIEMOP
122	* Pointer to an opcode decoder function.
123	*/
124
125	/** @def FNIEMOP_DEF
126	* Define an opcode decoder function.
127	*
128	* We're using macors for this so that adding and removing parameters as well as
129	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
130	*
131	* @param a_Name The function name.
132	*/
133
134
135	#if defined(__GNUC__) && defined(RT_ARCH_X86)
136	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PIEMCPU pIemCpu);
137	# define FNIEMOP_DEF(a_Name) \
138	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu)
139	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
140	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
141	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
142	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
143
144	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
145	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PIEMCPU pIemCpu);
146	# define FNIEMOP_DEF(a_Name) \
147	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu) RT_NO_THROW
148	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
149	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
150	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
151	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
152
153	#elif defined(__GNUC__)
154	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
155	# define FNIEMOP_DEF(a_Name) \
156	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu)
157	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
158	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
159	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
160	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
161
162	#else
163	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
164	# define FNIEMOP_DEF(a_Name) \
165	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu) RT_NO_THROW
166	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
167	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
168	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
169	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
170
171	#endif
172
173
174	/**
175	* Selector descriptor table entry as fetched by iemMemFetchSelDesc.
176	*/
177	typedef union IEMSELDESC
178	{
179	/** The legacy view. */
180	X86DESC Legacy;
181	/** The long mode view. */
182	X86DESC64 Long;
183	} IEMSELDESC;
184	/** Pointer to a selector descriptor table entry. */
185	typedef IEMSELDESC *PIEMSELDESC;
186
187
188	/*******************************************************************************
189	* Defined Constants And Macros *
190	*******************************************************************************/
191	/** @name IEM status codes.
192	*
193	* Not quite sure how this will play out in the end, just aliasing safe status
194	* codes for now.
195	*
196	* @{ */
197	#define VINF_IEM_RAISED_XCPT VINF_EM_RESCHEDULE
198	/** @} */
199
200	/** Temporary hack to disable the double execution. Will be removed in favor
201	* of a dedicated execution mode in EM. */
202	//#define IEM_VERIFICATION_MODE_NO_REM
203
204	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
205	* due to GCC lacking knowledge about the value range of a switch. */
206	#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
207
208	/**
209	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
210	* occation.
211	*/
212	#ifdef LOG_ENABLED
213	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
214	do { \
215	/Log/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
216	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
217	} while (0)
218	#else
219	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
220	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
221	#endif
222
223	/**
224	* Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
225	* occation using the supplied logger statement.
226	*
227	* @param a_LoggerArgs What to log on failure.
228	*/
229	#ifdef LOG_ENABLED
230	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
231	do { \
232	LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
233	/LogFunc(a_LoggerArgs);/ \
234	return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
235	} while (0)
236	#else
237	# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
238	return VERR_IEM_ASPECT_NOT_IMPLEMENTED
239	#endif
240
241	/**
242	* Call an opcode decoder function.
243	*
244	* We're using macors for this so that adding and removing parameters can be
245	* done as we please. See FNIEMOP_DEF.
246	*/
247	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pIemCpu)
248
249	/**
250	* Call a common opcode decoder function taking one extra argument.
251	*
252	* We're using macors for this so that adding and removing parameters can be
253	* done as we please. See FNIEMOP_DEF_1.
254	*/
255	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pIemCpu, a0)
256
257	/**
258	* Call a common opcode decoder function taking one extra argument.
259	*
260	* We're using macors for this so that adding and removing parameters can be
261	* done as we please. See FNIEMOP_DEF_1.
262	*/
263	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pIemCpu, a0, a1)
264
265	/**
266	* Check if we're currently executing in real or virtual 8086 mode.
267	*
268	* @returns @c true if it is, @c false if not.
269	* @param a_pIemCpu The IEM state of the current CPU.
270	*/
271	#define IEM_IS_REAL_OR_V86_MODE(a_pIemCpu) (CPUMIsGuestInRealOrV86ModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
272
273	/**
274	* Check if we're currently executing in virtual 8086 mode.
275	*
276	* @returns @c true if it is, @c false if not.
277	* @param a_pIemCpu The IEM state of the current CPU.
278	*/
279	#define IEM_IS_V86_MODE(a_pIemCpu) (CPUMIsGuestInV86ModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
280
281	/**
282	* Check if we're currently executing in long mode.
283	*
284	* @returns @c true if it is, @c false if not.
285	* @param a_pIemCpu The IEM state of the current CPU.
286	*/
287	#define IEM_IS_LONG_MODE(a_pIemCpu) (CPUMIsGuestInLongModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
288
289	/**
290	* Check if we're currently executing in real mode.
291	*
292	* @returns @c true if it is, @c false if not.
293	* @param a_pIemCpu The IEM state of the current CPU.
294	*/
295	#define IEM_IS_REAL_MODE(a_pIemCpu) (CPUMIsGuestInRealModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
296
297	/**
298	* Tests if an AMD CPUID feature (extended) is marked present - ECX.
299	*/
300	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX(a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, 0, (a_fEcx))
301
302	/**
303	* Tests if an AMD CPUID feature (extended) is marked present - EDX.
304	*/
305	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX(a_fEdx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0)
306
307	/**
308	* Tests if at least on of the specified AMD CPUID features (extended) are
309	* marked present.
310	*/
311	#define IEM_IS_AMD_CPUID_FEATURES_ANY_PRESENT(a_fEdx, a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), (a_fEcx))
312
313	/**
314	* Checks if an Intel CPUID feature is present.
315	*/
316	#define IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(a_fEdx) \
317	( ((a_fEdx) & (X86_CPUID_FEATURE_EDX_TSC \| 0)) \
318	\|\| iemRegIsIntelCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0) )
319
320	/**
321	* Checks if an Intel CPUID feature is present.
322	*/
323	#define IEM_IS_INTEL_CPUID_FEATURE_PRESENT_ECX(a_fEcx) \
324	( iemRegIsIntelCpuIdFeaturePresent(pIemCpu, 0, (a_fEcx)) )
325
326	/**
327	* Checks if an Intel CPUID feature is present in the host CPU.
328	*/
329	#define IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX_ON_HOST(a_fEdx) \
330	( (a_fEdx) & pIemCpu->fHostCpuIdStdFeaturesEdx )
331
332	/**
333	* Evaluates to true if we're presenting an Intel CPU to the guest.
334	*/
335	#define IEM_IS_GUEST_CPU_INTEL(a_pIemCpu) ( (a_pIemCpu)->enmCpuVendor == CPUMCPUVENDOR_INTEL )
336
337	/**
338	* Evaluates to true if we're presenting an AMD CPU to the guest.
339	*/
340	#define IEM_IS_GUEST_CPU_AMD(a_pIemCpu) ( (a_pIemCpu)->enmCpuVendor == CPUMCPUVENDOR_AMD )
341
342	/**
343	* Check if the address is canonical.
344	*/
345	#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
346
347
348	/*******************************************************************************
349	* Global Variables *
350	*******************************************************************************/
351	extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
352
353
354	/** Function table for the ADD instruction. */
355	static const IEMOPBINSIZES g_iemAImpl_add =
356	{
357	iemAImpl_add_u8, iemAImpl_add_u8_locked,
358	iemAImpl_add_u16, iemAImpl_add_u16_locked,
359	iemAImpl_add_u32, iemAImpl_add_u32_locked,
360	iemAImpl_add_u64, iemAImpl_add_u64_locked
361	};
362
363	/** Function table for the ADC instruction. */
364	static const IEMOPBINSIZES g_iemAImpl_adc =
365	{
366	iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
367	iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
368	iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
369	iemAImpl_adc_u64, iemAImpl_adc_u64_locked
370	};
371
372	/** Function table for the SUB instruction. */
373	static const IEMOPBINSIZES g_iemAImpl_sub =
374	{
375	iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
376	iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
377	iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
378	iemAImpl_sub_u64, iemAImpl_sub_u64_locked
379	};
380
381	/** Function table for the SBB instruction. */
382	static const IEMOPBINSIZES g_iemAImpl_sbb =
383	{
384	iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
385	iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
386	iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
387	iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
388	};
389
390	/** Function table for the OR instruction. */
391	static const IEMOPBINSIZES g_iemAImpl_or =
392	{
393	iemAImpl_or_u8, iemAImpl_or_u8_locked,
394	iemAImpl_or_u16, iemAImpl_or_u16_locked,
395	iemAImpl_or_u32, iemAImpl_or_u32_locked,
396	iemAImpl_or_u64, iemAImpl_or_u64_locked
397	};
398
399	/** Function table for the XOR instruction. */
400	static const IEMOPBINSIZES g_iemAImpl_xor =
401	{
402	iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
403	iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
404	iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
405	iemAImpl_xor_u64, iemAImpl_xor_u64_locked
406	};
407
408	/** Function table for the AND instruction. */
409	static const IEMOPBINSIZES g_iemAImpl_and =
410	{
411	iemAImpl_and_u8, iemAImpl_and_u8_locked,
412	iemAImpl_and_u16, iemAImpl_and_u16_locked,
413	iemAImpl_and_u32, iemAImpl_and_u32_locked,
414	iemAImpl_and_u64, iemAImpl_and_u64_locked
415	};
416
417	/** Function table for the CMP instruction.
418	* @remarks Making operand order ASSUMPTIONS.
419	*/
420	static const IEMOPBINSIZES g_iemAImpl_cmp =
421	{
422	iemAImpl_cmp_u8, NULL,
423	iemAImpl_cmp_u16, NULL,
424	iemAImpl_cmp_u32, NULL,
425	iemAImpl_cmp_u64, NULL
426	};
427
428	/** Function table for the TEST instruction.
429	* @remarks Making operand order ASSUMPTIONS.
430	*/
431	static const IEMOPBINSIZES g_iemAImpl_test =
432	{
433	iemAImpl_test_u8, NULL,
434	iemAImpl_test_u16, NULL,
435	iemAImpl_test_u32, NULL,
436	iemAImpl_test_u64, NULL
437	};
438
439	/** Function table for the BT instruction. */
440	static const IEMOPBINSIZES g_iemAImpl_bt =
441	{
442	NULL, NULL,
443	iemAImpl_bt_u16, NULL,
444	iemAImpl_bt_u32, NULL,
445	iemAImpl_bt_u64, NULL
446	};
447
448	/** Function table for the BTC instruction. */
449	static const IEMOPBINSIZES g_iemAImpl_btc =
450	{
451	NULL, NULL,
452	iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
453	iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
454	iemAImpl_btc_u64, iemAImpl_btc_u64_locked
455	};
456
457	/** Function table for the BTR instruction. */
458	static const IEMOPBINSIZES g_iemAImpl_btr =
459	{
460	NULL, NULL,
461	iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
462	iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
463	iemAImpl_btr_u64, iemAImpl_btr_u64_locked
464	};
465
466	/** Function table for the BTS instruction. */
467	static const IEMOPBINSIZES g_iemAImpl_bts =
468	{
469	NULL, NULL,
470	iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
471	iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
472	iemAImpl_bts_u64, iemAImpl_bts_u64_locked
473	};
474
475	/** Function table for the BSF instruction. */
476	static const IEMOPBINSIZES g_iemAImpl_bsf =
477	{
478	NULL, NULL,
479	iemAImpl_bsf_u16, NULL,
480	iemAImpl_bsf_u32, NULL,
481	iemAImpl_bsf_u64, NULL
482	};
483
484	/** Function table for the BSR instruction. */
485	static const IEMOPBINSIZES g_iemAImpl_bsr =
486	{
487	NULL, NULL,
488	iemAImpl_bsr_u16, NULL,
489	iemAImpl_bsr_u32, NULL,
490	iemAImpl_bsr_u64, NULL
491	};
492
493	/** Function table for the IMUL instruction. */
494	static const IEMOPBINSIZES g_iemAImpl_imul_two =
495	{
496	NULL, NULL,
497	iemAImpl_imul_two_u16, NULL,
498	iemAImpl_imul_two_u32, NULL,
499	iemAImpl_imul_two_u64, NULL
500	};
501
502	/** Group 1 /r lookup table. */
503	static const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
504	{
505	&g_iemAImpl_add,
506	&g_iemAImpl_or,
507	&g_iemAImpl_adc,
508	&g_iemAImpl_sbb,
509	&g_iemAImpl_and,
510	&g_iemAImpl_sub,
511	&g_iemAImpl_xor,
512	&g_iemAImpl_cmp
513	};
514
515	/** Function table for the INC instruction. */
516	static const IEMOPUNARYSIZES g_iemAImpl_inc =
517	{
518	iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
519	iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
520	iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
521	iemAImpl_inc_u64, iemAImpl_inc_u64_locked
522	};
523
524	/** Function table for the DEC instruction. */
525	static const IEMOPUNARYSIZES g_iemAImpl_dec =
526	{
527	iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
528	iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
529	iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
530	iemAImpl_dec_u64, iemAImpl_dec_u64_locked
531	};
532
533	/** Function table for the NEG instruction. */
534	static const IEMOPUNARYSIZES g_iemAImpl_neg =
535	{
536	iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
537	iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
538	iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
539	iemAImpl_neg_u64, iemAImpl_neg_u64_locked
540	};
541
542	/** Function table for the NOT instruction. */
543	static const IEMOPUNARYSIZES g_iemAImpl_not =
544	{
545	iemAImpl_not_u8, iemAImpl_not_u8_locked,
546	iemAImpl_not_u16, iemAImpl_not_u16_locked,
547	iemAImpl_not_u32, iemAImpl_not_u32_locked,
548	iemAImpl_not_u64, iemAImpl_not_u64_locked
549	};
550
551
552	/** Function table for the ROL instruction. */
553	static const IEMOPSHIFTSIZES g_iemAImpl_rol =
554	{
555	iemAImpl_rol_u8,
556	iemAImpl_rol_u16,
557	iemAImpl_rol_u32,
558	iemAImpl_rol_u64
559	};
560
561	/** Function table for the ROR instruction. */
562	static const IEMOPSHIFTSIZES g_iemAImpl_ror =
563	{
564	iemAImpl_ror_u8,
565	iemAImpl_ror_u16,
566	iemAImpl_ror_u32,
567	iemAImpl_ror_u64
568	};
569
570	/** Function table for the RCL instruction. */
571	static const IEMOPSHIFTSIZES g_iemAImpl_rcl =
572	{
573	iemAImpl_rcl_u8,
574	iemAImpl_rcl_u16,
575	iemAImpl_rcl_u32,
576	iemAImpl_rcl_u64
577	};
578
579	/** Function table for the RCR instruction. */
580	static const IEMOPSHIFTSIZES g_iemAImpl_rcr =
581	{
582	iemAImpl_rcr_u8,
583	iemAImpl_rcr_u16,
584	iemAImpl_rcr_u32,
585	iemAImpl_rcr_u64
586	};
587
588	/** Function table for the SHL instruction. */
589	static const IEMOPSHIFTSIZES g_iemAImpl_shl =
590	{
591	iemAImpl_shl_u8,
592	iemAImpl_shl_u16,
593	iemAImpl_shl_u32,
594	iemAImpl_shl_u64
595	};
596
597	/** Function table for the SHR instruction. */
598	static const IEMOPSHIFTSIZES g_iemAImpl_shr =
599	{
600	iemAImpl_shr_u8,
601	iemAImpl_shr_u16,
602	iemAImpl_shr_u32,
603	iemAImpl_shr_u64
604	};
605
606	/** Function table for the SAR instruction. */
607	static const IEMOPSHIFTSIZES g_iemAImpl_sar =
608	{
609	iemAImpl_sar_u8,
610	iemAImpl_sar_u16,
611	iemAImpl_sar_u32,
612	iemAImpl_sar_u64
613	};
614
615
616	/** Function table for the MUL instruction. */
617	static const IEMOPMULDIVSIZES g_iemAImpl_mul =
618	{
619	iemAImpl_mul_u8,
620	iemAImpl_mul_u16,
621	iemAImpl_mul_u32,
622	iemAImpl_mul_u64
623	};
624
625	/** Function table for the IMUL instruction working implicitly on rAX. */
626	static const IEMOPMULDIVSIZES g_iemAImpl_imul =
627	{
628	iemAImpl_imul_u8,
629	iemAImpl_imul_u16,
630	iemAImpl_imul_u32,
631	iemAImpl_imul_u64
632	};
633
634	/** Function table for the DIV instruction. */
635	static const IEMOPMULDIVSIZES g_iemAImpl_div =
636	{
637	iemAImpl_div_u8,
638	iemAImpl_div_u16,
639	iemAImpl_div_u32,
640	iemAImpl_div_u64
641	};
642
643	/** Function table for the MUL instruction. */
644	static const IEMOPMULDIVSIZES g_iemAImpl_idiv =
645	{
646	iemAImpl_idiv_u8,
647	iemAImpl_idiv_u16,
648	iemAImpl_idiv_u32,
649	iemAImpl_idiv_u64
650	};
651
652	/** Function table for the SHLD instruction */
653	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
654	{
655	iemAImpl_shld_u16,
656	iemAImpl_shld_u32,
657	iemAImpl_shld_u64,
658	};
659
660	/** Function table for the SHRD instruction */
661	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
662	{
663	iemAImpl_shrd_u16,
664	iemAImpl_shrd_u32,
665	iemAImpl_shrd_u64,
666	};
667
668
669	/** Function table for the PUNPCKLBW instruction */
670	static const IEMOPMEDIAF1L1 g_iemAImpl_punpcklbw = { iemAImpl_punpcklbw_u64, iemAImpl_punpcklbw_u128 };
671	/** Function table for the PUNPCKLBD instruction */
672	static const IEMOPMEDIAF1L1 g_iemAImpl_punpcklwd = { iemAImpl_punpcklwd_u64, iemAImpl_punpcklwd_u128 };
673	/** Function table for the PUNPCKLDQ instruction */
674	static const IEMOPMEDIAF1L1 g_iemAImpl_punpckldq = { iemAImpl_punpckldq_u64, iemAImpl_punpckldq_u128 };
675	/** Function table for the PUNPCKLQDQ instruction */
676	static const IEMOPMEDIAF1L1 g_iemAImpl_punpcklqdq = { NULL, iemAImpl_punpcklqdq_u128 };
677
678	/** Function table for the PUNPCKHBW instruction */
679	static const IEMOPMEDIAF1H1 g_iemAImpl_punpckhbw = { iemAImpl_punpckhbw_u64, iemAImpl_punpckhbw_u128 };
680	/** Function table for the PUNPCKHBD instruction */
681	static const IEMOPMEDIAF1H1 g_iemAImpl_punpckhwd = { iemAImpl_punpckhwd_u64, iemAImpl_punpckhwd_u128 };
682	/** Function table for the PUNPCKHDQ instruction */
683	static const IEMOPMEDIAF1H1 g_iemAImpl_punpckhdq = { iemAImpl_punpckhdq_u64, iemAImpl_punpckhdq_u128 };
684	/** Function table for the PUNPCKHQDQ instruction */
685	static const IEMOPMEDIAF1H1 g_iemAImpl_punpckhqdq = { NULL, iemAImpl_punpckhqdq_u128 };
686
687	/** Function table for the PXOR instruction */
688	static const IEMOPMEDIAF2 g_iemAImpl_pxor = { iemAImpl_pxor_u64, iemAImpl_pxor_u128 };
689	/** Function table for the PCMPEQB instruction */
690	static const IEMOPMEDIAF2 g_iemAImpl_pcmpeqb = { iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqb_u128 };
691	/** Function table for the PCMPEQW instruction */
692	static const IEMOPMEDIAF2 g_iemAImpl_pcmpeqw = { iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqw_u128 };
693	/** Function table for the PCMPEQD instruction */
694	static const IEMOPMEDIAF2 g_iemAImpl_pcmpeqd = { iemAImpl_pcmpeqd_u64, iemAImpl_pcmpeqd_u128 };
695
696
697	#if defined(IEM_VERIFICATION_MODE_MINIMAL) \|\| defined(IEM_LOG_MEMORY_WRITES)
698	/** What IEM just wrote. */
699	uint8_t g_abIemWrote[256];
700	/** How much IEM just wrote. */
701	size_t g_cbIemWrote;
702	#endif
703
704
705	/*******************************************************************************
706	* Internal Functions *
707	*******************************************************************************/
708	static VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PIEMCPU pIemCpu, uint16_t uErr);
709	static VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu);
710	static VBOXSTRICTRC iemRaiseTaskSwitchFault0(PIEMCPU pIemCpu);
711	static VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PIEMCPU pIemCpu, uint16_t uSel);
712	/static VBOXSTRICTRC iemRaiseSelectorNotPresent(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);/
713	static VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
714	static VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr);
715	static VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
716	static VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr);
717	static VBOXSTRICTRC iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr);
718	static VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu);
719	static VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL uSel);
720	static VBOXSTRICTRC iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
721	static VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel);
722	static VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
723	static VBOXSTRICTRC iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
724	static VBOXSTRICTRC iemRaiseAlignmentCheckException(PIEMCPU pIemCpu);
725	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
726	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess);
727	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
728	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
729	static VBOXSTRICTRC iemMemFetchSysU8(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
730	static VBOXSTRICTRC iemMemFetchSysU16(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
731	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
732	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
733	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode);
734	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt);
735	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp);
736	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp);
737	static VBOXSTRICTRC iemMemStackPushU32(PIEMCPU pIemCpu, uint32_t u32Value);
738	static VBOXSTRICTRC iemMemStackPushU16(PIEMCPU pIemCpu, uint16_t u16Value);
739	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel);
740	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg);
741
742	#if defined(IEM_VERIFICATION_MODE_FULL) && !defined(IEM_VERIFICATION_MODE_MINIMAL)
743	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu);
744	#endif
745	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue);
746	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue);
747
748
749
750	/**
751	* Sets the pass up status.
752	*
753	* @returns VINF_SUCCESS.
754	* @param pIemCpu The per CPU IEM state of the calling thread.
755	* @param rcPassUp The pass up status. Must be informational.
756	* VINF_SUCCESS is not allowed.
757	*/
758	static int iemSetPassUpStatus(PIEMCPU pIemCpu, VBOXSTRICTRC rcPassUp)
759	{
760	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
761
762	int32_t const rcOldPassUp = pIemCpu->rcPassUp;
763	if (rcOldPassUp == VINF_SUCCESS)
764	pIemCpu->rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
765	/* If both are EM scheduling codes, use EM priority rules. */
766	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
767	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
768	{
769	if (rcPassUp < rcOldPassUp)
770	{
771	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
772	pIemCpu->rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
773	}
774	else
775	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
776	}
777	/* Override EM scheduling with specific status code. */
778	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
779	{
780	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
781	pIemCpu->rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
782	}
783	/* Don't override specific status code, first come first served. */
784	else
785	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
786	return VINF_SUCCESS;
787	}
788
789
790	/**
791	* Initializes the execution state.
792	*
793	* @param pIemCpu The per CPU IEM state.
794	* @param fBypassHandlers Whether to bypass access handlers.
795	*/
796	DECLINLINE(void) iemInitExec(PIEMCPU pIemCpu, bool fBypassHandlers)
797	{
798	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
799	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
800
801	#if defined(VBOX_STRICT) && (defined(IEM_VERIFICATION_MODE_FULL) \|\| !defined(VBOX_WITH_RAW_MODE_NOT_R0))
802	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs));
803	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->ss));
804	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->es));
805	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->ds));
806	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->fs));
807	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->gs));
808	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->ldtr));
809	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->tr));
810	#endif
811
812	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
813	CPUMGuestLazyLoadHiddenCsAndSs(pVCpu);
814	#endif
815	pIemCpu->uCpl = CPUMGetGuestCPL(pVCpu);
816	IEMMODE enmMode = CPUMIsGuestIn64BitCodeEx(pCtx)
817	? IEMMODE_64BIT
818	: pCtx->cs.Attr.n.u1DefBig /** @todo check if this is correct... */
819	? IEMMODE_32BIT
820	: IEMMODE_16BIT;
821	pIemCpu->enmCpuMode = enmMode;
822	#ifdef VBOX_STRICT
823	pIemCpu->enmDefAddrMode = (IEMMODE)0xc0fe;
824	pIemCpu->enmEffAddrMode = (IEMMODE)0xc0fe;
825	pIemCpu->enmDefOpSize = (IEMMODE)0xc0fe;
826	pIemCpu->enmEffOpSize = (IEMMODE)0xc0fe;
827	pIemCpu->fPrefixes = (IEMMODE)0xfeedbeef;
828	pIemCpu->uRexReg = 127;
829	pIemCpu->uRexB = 127;
830	pIemCpu->uRexIndex = 127;
831	pIemCpu->iEffSeg = 127;
832	pIemCpu->offOpcode = 127;
833	pIemCpu->cbOpcode = 127;
834	#endif
835
836	pIemCpu->cActiveMappings = 0;
837	pIemCpu->iNextMapping = 0;
838	pIemCpu->rcPassUp = VINF_SUCCESS;
839	pIemCpu->fBypassHandlers = fBypassHandlers;
840	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
841	pIemCpu->fInPatchCode = pIemCpu->uCpl == 0
842	&& pCtx->cs.u64Base == 0
843	&& pCtx->cs.u32Limit == UINT32_MAX
844	&& PATMIsPatchGCAddr(IEMCPU_TO_VM(pIemCpu), pCtx->eip);
845	if (!pIemCpu->fInPatchCode)
846	CPUMRawLeave(pVCpu, CPUMCTX2CORE(pCtx), VINF_SUCCESS);
847	#endif
848	}
849
850
851	/**
852	* Initializes the decoder state.
853	*
854	* @param pIemCpu The per CPU IEM state.
855	* @param fBypassHandlers Whether to bypass access handlers.
856	*/
857	DECLINLINE(void) iemInitDecoder(PIEMCPU pIemCpu, bool fBypassHandlers)
858	{
859	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
860	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
861
862	#if defined(VBOX_STRICT) && (defined(IEM_VERIFICATION_MODE_FULL) \|\| !defined(VBOX_WITH_RAW_MODE_NOT_R0))
863	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs));
864	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->ss));
865	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->es));
866	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->ds));
867	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->fs));
868	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->gs));
869	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->ldtr));
870	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->tr));
871	#endif
872
873	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
874	CPUMGuestLazyLoadHiddenCsAndSs(pVCpu);
875	#endif
876	pIemCpu->uCpl = CPUMGetGuestCPL(pVCpu);
877	#ifdef IEM_VERIFICATION_MODE_FULL
878	if (pIemCpu->uInjectCpl != UINT8_MAX)
879	pIemCpu->uCpl = pIemCpu->uInjectCpl;
880	#endif
881	IEMMODE enmMode = CPUMIsGuestIn64BitCodeEx(pCtx)
882	? IEMMODE_64BIT
883	: pCtx->cs.Attr.n.u1DefBig /** @todo check if this is correct... */
884	? IEMMODE_32BIT
885	: IEMMODE_16BIT;
886	pIemCpu->enmCpuMode = enmMode;
887	pIemCpu->enmDefAddrMode = enmMode; /** @todo check if this is correct... */
888	pIemCpu->enmEffAddrMode = enmMode;
889	if (enmMode != IEMMODE_64BIT)
890	{
891	pIemCpu->enmDefOpSize = enmMode; /** @todo check if this is correct... */
892	pIemCpu->enmEffOpSize = enmMode;
893	}
894	else
895	{
896	pIemCpu->enmDefOpSize = IEMMODE_32BIT;
897	pIemCpu->enmEffOpSize = IEMMODE_32BIT;
898	}
899	pIemCpu->fPrefixes = 0;
900	pIemCpu->uRexReg = 0;
901	pIemCpu->uRexB = 0;
902	pIemCpu->uRexIndex = 0;
903	pIemCpu->iEffSeg = X86_SREG_DS;
904	pIemCpu->offOpcode = 0;
905	pIemCpu->cbOpcode = 0;
906	pIemCpu->cActiveMappings = 0;
907	pIemCpu->iNextMapping = 0;
908	pIemCpu->rcPassUp = VINF_SUCCESS;
909	pIemCpu->fBypassHandlers = fBypassHandlers;
910	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
911	pIemCpu->fInPatchCode = pIemCpu->uCpl == 0
912	&& pCtx->cs.u64Base == 0
913	&& pCtx->cs.u32Limit == UINT32_MAX
914	&& PATMIsPatchGCAddr(IEMCPU_TO_VM(pIemCpu), pCtx->eip);
915	if (!pIemCpu->fInPatchCode)
916	CPUMRawLeave(pVCpu, CPUMCTX2CORE(pCtx), VINF_SUCCESS);
917	#endif
918
919	#ifdef DBGFTRACE_ENABLED
920	switch (enmMode)
921	{
922	case IEMMODE_64BIT:
923	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pIemCpu->uCpl, pCtx->rip);
924	break;
925	case IEMMODE_32BIT:
926	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pIemCpu->uCpl, pCtx->cs.Sel, pCtx->eip);
927	break;
928	case IEMMODE_16BIT:
929	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pIemCpu->uCpl, pCtx->cs.Sel, pCtx->eip);
930	break;
931	}
932	#endif
933	}
934
935
936	/**
937	* Prefetch opcodes the first time when starting executing.
938	*
939	* @returns Strict VBox status code.
940	* @param pIemCpu The IEM state.
941	* @param fBypassHandlers Whether to bypass access handlers.
942	*/
943	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PIEMCPU pIemCpu, bool fBypassHandlers)
944	{
945	#ifdef IEM_VERIFICATION_MODE_FULL
946	uint8_t const cbOldOpcodes = pIemCpu->cbOpcode;
947	#endif
948	iemInitDecoder(pIemCpu, fBypassHandlers);
949
950	/*
951	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
952	*
953	* First translate CS:rIP to a physical address.
954	*/
955	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
956	uint32_t cbToTryRead;
957	RTGCPTR GCPtrPC;
958	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
959	{
960	cbToTryRead = PAGE_SIZE;
961	GCPtrPC = pCtx->rip;
962	if (!IEM_IS_CANONICAL(GCPtrPC))
963	return iemRaiseGeneralProtectionFault0(pIemCpu);
964	cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
965	}
966	else
967	{
968	uint32_t GCPtrPC32 = pCtx->eip;
969	AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
970	if (GCPtrPC32 > pCtx->cs.u32Limit)
971	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
972	cbToTryRead = pCtx->cs.u32Limit - GCPtrPC32 + 1;
973	if (!cbToTryRead) /* overflowed */
974	{
975	Assert(GCPtrPC32 == 0); Assert(pCtx->cs.u32Limit == UINT32_MAX);
976	cbToTryRead = UINT32_MAX;
977	}
978	GCPtrPC = pCtx->cs.u64Base + GCPtrPC32;
979	}
980
981	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
982	/* Allow interpretation of patch manager code blocks since they can for
983	instance throw #PFs for perfectly good reasons. */
984	if (pIemCpu->fInPatchCode)
985	{
986	size_t cbRead = 0;
987	int rc = PATMReadPatchCode(IEMCPU_TO_VM(pIemCpu), GCPtrPC, pIemCpu->abOpcode, sizeof(pIemCpu->abOpcode), &cbRead);
988	AssertRCReturn(rc, rc);
989	pIemCpu->cbOpcode = (uint8_t)cbRead; Assert(pIemCpu->cbOpcode == cbRead); Assert(cbRead > 0);
990	return VINF_SUCCESS;
991	}
992	#endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
993
994	RTGCPHYS GCPhys;
995	uint64_t fFlags;
996	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrPC, &fFlags, &GCPhys);
997	if (RT_FAILURE(rc))
998	{
999	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
1000	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
1001	}
1002	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
1003	{
1004	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
1005	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1006	}
1007	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
1008	{
1009	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
1010	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1011	}
1012	GCPhys \|= GCPtrPC & PAGE_OFFSET_MASK;
1013	/** @todo Check reserved bits and such stuff. PGM is better at doing
1014	* that, so do it when implementing the guest virtual address
1015	* TLB... */
1016
1017	#ifdef IEM_VERIFICATION_MODE_FULL
1018	/*
1019	* Optimistic optimization: Use unconsumed opcode bytes from the previous
1020	* instruction.
1021	*/
1022	/** @todo optimize this differently by not using PGMPhysRead. */
1023	RTGCPHYS const offPrevOpcodes = GCPhys - pIemCpu->GCPhysOpcodes;
1024	pIemCpu->GCPhysOpcodes = GCPhys;
1025	if ( offPrevOpcodes < cbOldOpcodes
1026	&& PAGE_SIZE - (GCPhys & PAGE_OFFSET_MASK) > sizeof(pIemCpu->abOpcode))
1027	{
1028	uint8_t cbNew = cbOldOpcodes - (uint8_t)offPrevOpcodes;
1029	memmove(&pIemCpu->abOpcode[0], &pIemCpu->abOpcode[offPrevOpcodes], cbNew);
1030	pIemCpu->cbOpcode = cbNew;
1031	return VINF_SUCCESS;
1032	}
1033	#endif
1034
1035	/*
1036	* Read the bytes at this address.
1037	*/
1038	PVM pVM = IEMCPU_TO_VM(pIemCpu);
1039	#if defined(IN_RING3) && defined(VBOX_WITH_RAW_MODE_NOT_R0)
1040	size_t cbActual;
1041	if ( PATMIsEnabled(pVM)
1042	&& RT_SUCCESS(PATMR3ReadOrgInstr(pVM, GCPtrPC, pIemCpu->abOpcode, sizeof(pIemCpu->abOpcode), &cbActual)))
1043	{
1044	Log4(("decode - Read %u unpatched bytes at %RGv\n", cbActual, GCPtrPC));
1045	Assert(cbActual > 0);
1046	pIemCpu->cbOpcode = (uint8_t)cbActual;
1047	}
1048	else
1049	#endif
1050	{
1051	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
1052	if (cbToTryRead > cbLeftOnPage)
1053	cbToTryRead = cbLeftOnPage;
1054	if (cbToTryRead > sizeof(pIemCpu->abOpcode))
1055	cbToTryRead = sizeof(pIemCpu->abOpcode);
1056
1057	if (!pIemCpu->fBypassHandlers)
1058	rc = PGMPhysRead(pVM, GCPhys, pIemCpu->abOpcode, cbToTryRead);
1059	else
1060	rc = PGMPhysSimpleReadGCPhys(pVM, pIemCpu->abOpcode, GCPhys, cbToTryRead);
1061	if (rc != VINF_SUCCESS)
1062	{
1063	/** @todo status code handling */
1064	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
1065	GCPtrPC, GCPhys, rc, cbToTryRead));
1066	return rc;
1067	}
1068	pIemCpu->cbOpcode = cbToTryRead;
1069	}
1070
1071	return VINF_SUCCESS;
1072	}
1073
1074
1075	/**
1076	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
1077	* exception if it fails.
1078	*
1079	* @returns Strict VBox status code.
1080	* @param pIemCpu The IEM state.
1081	* @param cbMin The minimum number of bytes relative offOpcode
1082	* that must be read.
1083	*/
1084	static VBOXSTRICTRC iemOpcodeFetchMoreBytes(PIEMCPU pIemCpu, size_t cbMin)
1085	{
1086	/*
1087	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1088	*
1089	* First translate CS:rIP to a physical address.
1090	*/
1091	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
1092	uint8_t cbLeft = pIemCpu->cbOpcode - pIemCpu->offOpcode; Assert(cbLeft < cbMin);
1093	uint32_t cbToTryRead;
1094	RTGCPTR GCPtrNext;
1095	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
1096	{
1097	cbToTryRead = PAGE_SIZE;
1098	GCPtrNext = pCtx->rip + pIemCpu->cbOpcode;
1099	if (!IEM_IS_CANONICAL(GCPtrNext))
1100	return iemRaiseGeneralProtectionFault0(pIemCpu);
1101	}
1102	else
1103	{
1104	uint32_t GCPtrNext32 = pCtx->eip;
1105	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
1106	GCPtrNext32 += pIemCpu->cbOpcode;
1107	if (GCPtrNext32 > pCtx->cs.u32Limit)
1108	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1109	cbToTryRead = pCtx->cs.u32Limit - GCPtrNext32 + 1;
1110	if (!cbToTryRead) /* overflowed */
1111	{
1112	Assert(GCPtrNext32 == 0); Assert(pCtx->cs.u32Limit == UINT32_MAX);
1113	cbToTryRead = UINT32_MAX;
1114	/** @todo check out wrapping around the code segment. */
1115	}
1116	if (cbToTryRead < cbMin - cbLeft)
1117	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1118	GCPtrNext = pCtx->cs.u64Base + GCPtrNext32;
1119	}
1120
1121	/* Only read up to the end of the page, and make sure we don't read more
1122	than the opcode buffer can hold. */
1123	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
1124	if (cbToTryRead > cbLeftOnPage)
1125	cbToTryRead = cbLeftOnPage;
1126	if (cbToTryRead > sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode)
1127	cbToTryRead = sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode;
1128	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1129
1130	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
1131	/* Allow interpretation of patch manager code blocks since they can for
1132	instance throw #PFs for perfectly good reasons. */
1133	if (pIemCpu->fInPatchCode)
1134	{
1135	size_t cbRead = 0;
1136	int rc = PATMReadPatchCode(IEMCPU_TO_VM(pIemCpu), GCPtrNext, pIemCpu->abOpcode, cbToTryRead, &cbRead);
1137	AssertRCReturn(rc, rc);
1138	pIemCpu->cbOpcode = (uint8_t)cbRead; Assert(pIemCpu->cbOpcode == cbRead); Assert(cbRead > 0);
1139	return VINF_SUCCESS;
1140	}
1141	#endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
1142
1143	RTGCPHYS GCPhys;
1144	uint64_t fFlags;
1145	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrNext, &fFlags, &GCPhys);
1146	if (RT_FAILURE(rc))
1147	{
1148	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1149	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1150	}
1151	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
1152	{
1153	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1154	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1155	}
1156	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
1157	{
1158	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1159	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1160	}
1161	GCPhys \|= GCPtrNext & PAGE_OFFSET_MASK;
1162	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pIemCpu->cbOpcode));
1163	/** @todo Check reserved bits and such stuff. PGM is better at doing
1164	* that, so do it when implementing the guest virtual address
1165	* TLB... */
1166
1167	/*
1168	* Read the bytes at this address.
1169	*
1170	* We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1171	* and since PATM should only patch the start of an instruction there
1172	* should be no need to check again here.
1173	*/
1174	if (!pIemCpu->fBypassHandlers)
1175	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, &pIemCpu->abOpcode[pIemCpu->cbOpcode], cbToTryRead);
1176	else
1177	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), &pIemCpu->abOpcode[pIemCpu->cbOpcode], GCPhys, cbToTryRead);
1178	if (rc != VINF_SUCCESS)
1179	{
1180	/** @todo status code handling */
1181	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
1182	return rc;
1183	}
1184	pIemCpu->cbOpcode += cbToTryRead;
1185	Log5(("%.*Rhxs\n", pIemCpu->cbOpcode, pIemCpu->abOpcode));
1186
1187	return VINF_SUCCESS;
1188	}
1189
1190
1191	/**
1192	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
1193	*
1194	* @returns Strict VBox status code.
1195	* @param pIemCpu The IEM state.
1196	* @param pb Where to return the opcode byte.
1197	*/
1198	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PIEMCPU pIemCpu, uint8_t *pb)
1199	{
1200	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 1);
1201	if (rcStrict == VINF_SUCCESS)
1202	{
1203	uint8_t offOpcode = pIemCpu->offOpcode;
1204	*pb = pIemCpu->abOpcode[offOpcode];
1205	pIemCpu->offOpcode = offOpcode + 1;
1206	}
1207	else
1208	*pb = 0;
1209	return rcStrict;
1210	}
1211
1212
1213	/**
1214	* Fetches the next opcode byte.
1215	*
1216	* @returns Strict VBox status code.
1217	* @param pIemCpu The IEM state.
1218	* @param pu8 Where to return the opcode byte.
1219	*/
1220	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PIEMCPU pIemCpu, uint8_t *pu8)
1221	{
1222	uint8_t const offOpcode = pIemCpu->offOpcode;
1223	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
1224	return iemOpcodeGetNextU8Slow(pIemCpu, pu8);
1225
1226	*pu8 = pIemCpu->abOpcode[offOpcode];
1227	pIemCpu->offOpcode = offOpcode + 1;
1228	return VINF_SUCCESS;
1229	}
1230
1231
1232	/**
1233	* Fetches the next opcode byte, returns automatically on failure.
1234	*
1235	* @param a_pu8 Where to return the opcode byte.
1236	* @remark Implicitly references pIemCpu.
1237	*/
1238	#define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
1239	do \
1240	{ \
1241	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pIemCpu, (a_pu8)); \
1242	if (rcStrict2 != VINF_SUCCESS) \
1243	return rcStrict2; \
1244	} while (0)
1245
1246
1247	/**
1248	* Fetches the next signed byte from the opcode stream.
1249	*
1250	* @returns Strict VBox status code.
1251	* @param pIemCpu The IEM state.
1252	* @param pi8 Where to return the signed byte.
1253	*/
1254	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PIEMCPU pIemCpu, int8_t *pi8)
1255	{
1256	return iemOpcodeGetNextU8(pIemCpu, (uint8_t *)pi8);
1257	}
1258
1259
1260	/**
1261	* Fetches the next signed byte from the opcode stream, returning automatically
1262	* on failure.
1263	*
1264	* @param pi8 Where to return the signed byte.
1265	* @remark Implicitly references pIemCpu.
1266	*/
1267	#define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
1268	do \
1269	{ \
1270	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pIemCpu, (a_pi8)); \
1271	if (rcStrict2 != VINF_SUCCESS) \
1272	return rcStrict2; \
1273	} while (0)
1274
1275
1276	/**
1277	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
1278	*
1279	* @returns Strict VBox status code.
1280	* @param pIemCpu The IEM state.
1281	* @param pu16 Where to return the opcode dword.
1282	*/
1283	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
1284	{
1285	uint8_t u8;
1286	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pIemCpu, &u8);
1287	if (rcStrict == VINF_SUCCESS)
1288	*pu16 = (int8_t)u8;
1289	return rcStrict;
1290	}
1291
1292
1293	/**
1294	* Fetches the next signed byte from the opcode stream, extending it to
1295	* unsigned 16-bit.
1296	*
1297	* @returns Strict VBox status code.
1298	* @param pIemCpu The IEM state.
1299	* @param pu16 Where to return the unsigned word.
1300	*/
1301	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PIEMCPU pIemCpu, uint16_t *pu16)
1302	{
1303	uint8_t const offOpcode = pIemCpu->offOpcode;
1304	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
1305	return iemOpcodeGetNextS8SxU16Slow(pIemCpu, pu16);
1306
1307	*pu16 = (int8_t)pIemCpu->abOpcode[offOpcode];
1308	pIemCpu->offOpcode = offOpcode + 1;
1309	return VINF_SUCCESS;
1310	}
1311
1312
1313	/**
1314	* Fetches the next signed byte from the opcode stream and sign-extending it to
1315	* a word, returning automatically on failure.
1316	*
1317	* @param pu16 Where to return the word.
1318	* @remark Implicitly references pIemCpu.
1319	*/
1320	#define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
1321	do \
1322	{ \
1323	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pIemCpu, (a_pu16)); \
1324	if (rcStrict2 != VINF_SUCCESS) \
1325	return rcStrict2; \
1326	} while (0)
1327
1328
1329	/**
1330	* Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
1331	*
1332	* @returns Strict VBox status code.
1333	* @param pIemCpu The IEM state.
1334	* @param pu32 Where to return the opcode dword.
1335	*/
1336	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS8SxU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1337	{
1338	uint8_t u8;
1339	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pIemCpu, &u8);
1340	if (rcStrict == VINF_SUCCESS)
1341	*pu32 = (int8_t)u8;
1342	return rcStrict;
1343	}
1344
1345
1346	/**
1347	* Fetches the next signed byte from the opcode stream, extending it to
1348	* unsigned 32-bit.
1349	*
1350	* @returns Strict VBox status code.
1351	* @param pIemCpu The IEM state.
1352	* @param pu32 Where to return the unsigned dword.
1353	*/
1354	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PIEMCPU pIemCpu, uint32_t *pu32)
1355	{
1356	uint8_t const offOpcode = pIemCpu->offOpcode;
1357	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
1358	return iemOpcodeGetNextS8SxU32Slow(pIemCpu, pu32);
1359
1360	*pu32 = (int8_t)pIemCpu->abOpcode[offOpcode];
1361	pIemCpu->offOpcode = offOpcode + 1;
1362	return VINF_SUCCESS;
1363	}
1364
1365
1366	/**
1367	* Fetches the next signed byte from the opcode stream and sign-extending it to
1368	* a word, returning automatically on failure.
1369	*
1370	* @param pu32 Where to return the word.
1371	* @remark Implicitly references pIemCpu.
1372	*/
1373	#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
1374	do \
1375	{ \
1376	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pIemCpu, (a_pu32)); \
1377	if (rcStrict2 != VINF_SUCCESS) \
1378	return rcStrict2; \
1379	} while (0)
1380
1381
1382	/**
1383	* Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
1384	*
1385	* @returns Strict VBox status code.
1386	* @param pIemCpu The IEM state.
1387	* @param pu64 Where to return the opcode qword.
1388	*/
1389	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS8SxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1390	{
1391	uint8_t u8;
1392	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pIemCpu, &u8);
1393	if (rcStrict == VINF_SUCCESS)
1394	*pu64 = (int8_t)u8;
1395	return rcStrict;
1396	}
1397
1398
1399	/**
1400	* Fetches the next signed byte from the opcode stream, extending it to
1401	* unsigned 64-bit.
1402	*
1403	* @returns Strict VBox status code.
1404	* @param pIemCpu The IEM state.
1405	* @param pu64 Where to return the unsigned qword.
1406	*/
1407	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1408	{
1409	uint8_t const offOpcode = pIemCpu->offOpcode;
1410	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
1411	return iemOpcodeGetNextS8SxU64Slow(pIemCpu, pu64);
1412
1413	*pu64 = (int8_t)pIemCpu->abOpcode[offOpcode];
1414	pIemCpu->offOpcode = offOpcode + 1;
1415	return VINF_SUCCESS;
1416	}
1417
1418
1419	/**
1420	* Fetches the next signed byte from the opcode stream and sign-extending it to
1421	* a word, returning automatically on failure.
1422	*
1423	* @param pu64 Where to return the word.
1424	* @remark Implicitly references pIemCpu.
1425	*/
1426	#define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
1427	do \
1428	{ \
1429	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pIemCpu, (a_pu64)); \
1430	if (rcStrict2 != VINF_SUCCESS) \
1431	return rcStrict2; \
1432	} while (0)
1433
1434
1435	/**
1436	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
1437	*
1438	* @returns Strict VBox status code.
1439	* @param pIemCpu The IEM state.
1440	* @param pu16 Where to return the opcode word.
1441	*/
1442	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
1443	{
1444	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1445	if (rcStrict == VINF_SUCCESS)
1446	{
1447	uint8_t offOpcode = pIemCpu->offOpcode;
1448	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1449	pIemCpu->offOpcode = offOpcode + 2;
1450	}
1451	else
1452	*pu16 = 0;
1453	return rcStrict;
1454	}
1455
1456
1457	/**
1458	* Fetches the next opcode word.
1459	*
1460	* @returns Strict VBox status code.
1461	* @param pIemCpu The IEM state.
1462	* @param pu16 Where to return the opcode word.
1463	*/
1464	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PIEMCPU pIemCpu, uint16_t *pu16)
1465	{
1466	uint8_t const offOpcode = pIemCpu->offOpcode;
1467	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1468	return iemOpcodeGetNextU16Slow(pIemCpu, pu16);
1469
1470	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1471	pIemCpu->offOpcode = offOpcode + 2;
1472	return VINF_SUCCESS;
1473	}
1474
1475
1476	/**
1477	* Fetches the next opcode word, returns automatically on failure.
1478	*
1479	* @param a_pu16 Where to return the opcode word.
1480	* @remark Implicitly references pIemCpu.
1481	*/
1482	#define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
1483	do \
1484	{ \
1485	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pIemCpu, (a_pu16)); \
1486	if (rcStrict2 != VINF_SUCCESS) \
1487	return rcStrict2; \
1488	} while (0)
1489
1490
1491	/**
1492	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1493	*
1494	* @returns Strict VBox status code.
1495	* @param pIemCpu The IEM state.
1496	* @param pu32 Where to return the opcode double word.
1497	*/
1498	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1499	{
1500	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1501	if (rcStrict == VINF_SUCCESS)
1502	{
1503	uint8_t offOpcode = pIemCpu->offOpcode;
1504	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1505	pIemCpu->offOpcode = offOpcode + 2;
1506	}
1507	else
1508	*pu32 = 0;
1509	return rcStrict;
1510	}
1511
1512
1513	/**
1514	* Fetches the next opcode word, zero extending it to a double word.
1515	*
1516	* @returns Strict VBox status code.
1517	* @param pIemCpu The IEM state.
1518	* @param pu32 Where to return the opcode double word.
1519	*/
1520	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PIEMCPU pIemCpu, uint32_t *pu32)
1521	{
1522	uint8_t const offOpcode = pIemCpu->offOpcode;
1523	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1524	return iemOpcodeGetNextU16ZxU32Slow(pIemCpu, pu32);
1525
1526	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1527	pIemCpu->offOpcode = offOpcode + 2;
1528	return VINF_SUCCESS;
1529	}
1530
1531
1532	/**
1533	* Fetches the next opcode word and zero extends it to a double word, returns
1534	* automatically on failure.
1535	*
1536	* @param a_pu32 Where to return the opcode double word.
1537	* @remark Implicitly references pIemCpu.
1538	*/
1539	#define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1540	do \
1541	{ \
1542	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pIemCpu, (a_pu32)); \
1543	if (rcStrict2 != VINF_SUCCESS) \
1544	return rcStrict2; \
1545	} while (0)
1546
1547
1548	/**
1549	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1550	*
1551	* @returns Strict VBox status code.
1552	* @param pIemCpu The IEM state.
1553	* @param pu64 Where to return the opcode quad word.
1554	*/
1555	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1556	{
1557	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1558	if (rcStrict == VINF_SUCCESS)
1559	{
1560	uint8_t offOpcode = pIemCpu->offOpcode;
1561	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1562	pIemCpu->offOpcode = offOpcode + 2;
1563	}
1564	else
1565	*pu64 = 0;
1566	return rcStrict;
1567	}
1568
1569
1570	/**
1571	* Fetches the next opcode word, zero extending it to a quad word.
1572	*
1573	* @returns Strict VBox status code.
1574	* @param pIemCpu The IEM state.
1575	* @param pu64 Where to return the opcode quad word.
1576	*/
1577	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1578	{
1579	uint8_t const offOpcode = pIemCpu->offOpcode;
1580	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1581	return iemOpcodeGetNextU16ZxU64Slow(pIemCpu, pu64);
1582
1583	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1584	pIemCpu->offOpcode = offOpcode + 2;
1585	return VINF_SUCCESS;
1586	}
1587
1588
1589	/**
1590	* Fetches the next opcode word and zero extends it to a quad word, returns
1591	* automatically on failure.
1592	*
1593	* @param a_pu64 Where to return the opcode quad word.
1594	* @remark Implicitly references pIemCpu.
1595	*/
1596	#define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1597	do \
1598	{ \
1599	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pIemCpu, (a_pu64)); \
1600	if (rcStrict2 != VINF_SUCCESS) \
1601	return rcStrict2; \
1602	} while (0)
1603
1604
1605	/**
1606	* Fetches the next signed word from the opcode stream.
1607	*
1608	* @returns Strict VBox status code.
1609	* @param pIemCpu The IEM state.
1610	* @param pi16 Where to return the signed word.
1611	*/
1612	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PIEMCPU pIemCpu, int16_t *pi16)
1613	{
1614	return iemOpcodeGetNextU16(pIemCpu, (uint16_t *)pi16);
1615	}
1616
1617
1618	/**
1619	* Fetches the next signed word from the opcode stream, returning automatically
1620	* on failure.
1621	*
1622	* @param pi16 Where to return the signed word.
1623	* @remark Implicitly references pIemCpu.
1624	*/
1625	#define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1626	do \
1627	{ \
1628	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pIemCpu, (a_pi16)); \
1629	if (rcStrict2 != VINF_SUCCESS) \
1630	return rcStrict2; \
1631	} while (0)
1632
1633
1634	/**
1635	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1636	*
1637	* @returns Strict VBox status code.
1638	* @param pIemCpu The IEM state.
1639	* @param pu32 Where to return the opcode dword.
1640	*/
1641	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1642	{
1643	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1644	if (rcStrict == VINF_SUCCESS)
1645	{
1646	uint8_t offOpcode = pIemCpu->offOpcode;
1647	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1648	pIemCpu->abOpcode[offOpcode + 1],
1649	pIemCpu->abOpcode[offOpcode + 2],
1650	pIemCpu->abOpcode[offOpcode + 3]);
1651	pIemCpu->offOpcode = offOpcode + 4;
1652	}
1653	else
1654	*pu32 = 0;
1655	return rcStrict;
1656	}
1657
1658
1659	/**
1660	* Fetches the next opcode dword.
1661	*
1662	* @returns Strict VBox status code.
1663	* @param pIemCpu The IEM state.
1664	* @param pu32 Where to return the opcode double word.
1665	*/
1666	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PIEMCPU pIemCpu, uint32_t *pu32)
1667	{
1668	uint8_t const offOpcode = pIemCpu->offOpcode;
1669	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1670	return iemOpcodeGetNextU32Slow(pIemCpu, pu32);
1671
1672	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1673	pIemCpu->abOpcode[offOpcode + 1],
1674	pIemCpu->abOpcode[offOpcode + 2],
1675	pIemCpu->abOpcode[offOpcode + 3]);
1676	pIemCpu->offOpcode = offOpcode + 4;
1677	return VINF_SUCCESS;
1678	}
1679
1680
1681	/**
1682	* Fetches the next opcode dword, returns automatically on failure.
1683	*
1684	* @param a_pu32 Where to return the opcode dword.
1685	* @remark Implicitly references pIemCpu.
1686	*/
1687	#define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1688	do \
1689	{ \
1690	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pIemCpu, (a_pu32)); \
1691	if (rcStrict2 != VINF_SUCCESS) \
1692	return rcStrict2; \
1693	} while (0)
1694
1695
1696	/**
1697	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1698	*
1699	* @returns Strict VBox status code.
1700	* @param pIemCpu The IEM state.
1701	* @param pu32 Where to return the opcode dword.
1702	*/
1703	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1704	{
1705	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1706	if (rcStrict == VINF_SUCCESS)
1707	{
1708	uint8_t offOpcode = pIemCpu->offOpcode;
1709	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1710	pIemCpu->abOpcode[offOpcode + 1],
1711	pIemCpu->abOpcode[offOpcode + 2],
1712	pIemCpu->abOpcode[offOpcode + 3]);
1713	pIemCpu->offOpcode = offOpcode + 4;
1714	}
1715	else
1716	*pu64 = 0;
1717	return rcStrict;
1718	}
1719
1720
1721	/**
1722	* Fetches the next opcode dword, zero extending it to a quad word.
1723	*
1724	* @returns Strict VBox status code.
1725	* @param pIemCpu The IEM state.
1726	* @param pu64 Where to return the opcode quad word.
1727	*/
1728	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1729	{
1730	uint8_t const offOpcode = pIemCpu->offOpcode;
1731	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1732	return iemOpcodeGetNextU32ZxU64Slow(pIemCpu, pu64);
1733
1734	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1735	pIemCpu->abOpcode[offOpcode + 1],
1736	pIemCpu->abOpcode[offOpcode + 2],
1737	pIemCpu->abOpcode[offOpcode + 3]);
1738	pIemCpu->offOpcode = offOpcode + 4;
1739	return VINF_SUCCESS;
1740	}
1741
1742
1743	/**
1744	* Fetches the next opcode dword and zero extends it to a quad word, returns
1745	* automatically on failure.
1746	*
1747	* @param a_pu64 Where to return the opcode quad word.
1748	* @remark Implicitly references pIemCpu.
1749	*/
1750	#define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1751	do \
1752	{ \
1753	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pIemCpu, (a_pu64)); \
1754	if (rcStrict2 != VINF_SUCCESS) \
1755	return rcStrict2; \
1756	} while (0)
1757
1758
1759	/**
1760	* Fetches the next signed double word from the opcode stream.
1761	*
1762	* @returns Strict VBox status code.
1763	* @param pIemCpu The IEM state.
1764	* @param pi32 Where to return the signed double word.
1765	*/
1766	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PIEMCPU pIemCpu, int32_t *pi32)
1767	{
1768	return iemOpcodeGetNextU32(pIemCpu, (uint32_t *)pi32);
1769	}
1770
1771	/**
1772	* Fetches the next signed double word from the opcode stream, returning
1773	* automatically on failure.
1774	*
1775	* @param pi32 Where to return the signed double word.
1776	* @remark Implicitly references pIemCpu.
1777	*/
1778	#define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1779	do \
1780	{ \
1781	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pIemCpu, (a_pi32)); \
1782	if (rcStrict2 != VINF_SUCCESS) \
1783	return rcStrict2; \
1784	} while (0)
1785
1786
1787	/**
1788	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1789	*
1790	* @returns Strict VBox status code.
1791	* @param pIemCpu The IEM state.
1792	* @param pu64 Where to return the opcode qword.
1793	*/
1794	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1795	{
1796	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1797	if (rcStrict == VINF_SUCCESS)
1798	{
1799	uint8_t offOpcode = pIemCpu->offOpcode;
1800	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1801	pIemCpu->abOpcode[offOpcode + 1],
1802	pIemCpu->abOpcode[offOpcode + 2],
1803	pIemCpu->abOpcode[offOpcode + 3]);
1804	pIemCpu->offOpcode = offOpcode + 4;
1805	}
1806	else
1807	*pu64 = 0;
1808	return rcStrict;
1809	}
1810
1811
1812	/**
1813	* Fetches the next opcode dword, sign extending it into a quad word.
1814	*
1815	* @returns Strict VBox status code.
1816	* @param pIemCpu The IEM state.
1817	* @param pu64 Where to return the opcode quad word.
1818	*/
1819	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1820	{
1821	uint8_t const offOpcode = pIemCpu->offOpcode;
1822	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1823	return iemOpcodeGetNextS32SxU64Slow(pIemCpu, pu64);
1824
1825	int32_t i32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1826	pIemCpu->abOpcode[offOpcode + 1],
1827	pIemCpu->abOpcode[offOpcode + 2],
1828	pIemCpu->abOpcode[offOpcode + 3]);
1829	*pu64 = i32;
1830	pIemCpu->offOpcode = offOpcode + 4;
1831	return VINF_SUCCESS;
1832	}
1833
1834
1835	/**
1836	* Fetches the next opcode double word and sign extends it to a quad word,
1837	* returns automatically on failure.
1838	*
1839	* @param a_pu64 Where to return the opcode quad word.
1840	* @remark Implicitly references pIemCpu.
1841	*/
1842	#define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1843	do \
1844	{ \
1845	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pIemCpu, (a_pu64)); \
1846	if (rcStrict2 != VINF_SUCCESS) \
1847	return rcStrict2; \
1848	} while (0)
1849
1850
1851	/**
1852	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1853	*
1854	* @returns Strict VBox status code.
1855	* @param pIemCpu The IEM state.
1856	* @param pu64 Where to return the opcode qword.
1857	*/
1858	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1859	{
1860	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 8);
1861	if (rcStrict == VINF_SUCCESS)
1862	{
1863	uint8_t offOpcode = pIemCpu->offOpcode;
1864	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1865	pIemCpu->abOpcode[offOpcode + 1],
1866	pIemCpu->abOpcode[offOpcode + 2],
1867	pIemCpu->abOpcode[offOpcode + 3],
1868	pIemCpu->abOpcode[offOpcode + 4],
1869	pIemCpu->abOpcode[offOpcode + 5],
1870	pIemCpu->abOpcode[offOpcode + 6],
1871	pIemCpu->abOpcode[offOpcode + 7]);
1872	pIemCpu->offOpcode = offOpcode + 8;
1873	}
1874	else
1875	*pu64 = 0;
1876	return rcStrict;
1877	}
1878
1879
1880	/**
1881	* Fetches the next opcode qword.
1882	*
1883	* @returns Strict VBox status code.
1884	* @param pIemCpu The IEM state.
1885	* @param pu64 Where to return the opcode qword.
1886	*/
1887	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PIEMCPU pIemCpu, uint64_t *pu64)
1888	{
1889	uint8_t const offOpcode = pIemCpu->offOpcode;
1890	if (RT_UNLIKELY(offOpcode + 8 > pIemCpu->cbOpcode))
1891	return iemOpcodeGetNextU64Slow(pIemCpu, pu64);
1892
1893	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1894	pIemCpu->abOpcode[offOpcode + 1],
1895	pIemCpu->abOpcode[offOpcode + 2],
1896	pIemCpu->abOpcode[offOpcode + 3],
1897	pIemCpu->abOpcode[offOpcode + 4],
1898	pIemCpu->abOpcode[offOpcode + 5],
1899	pIemCpu->abOpcode[offOpcode + 6],
1900	pIemCpu->abOpcode[offOpcode + 7]);
1901	pIemCpu->offOpcode = offOpcode + 8;
1902	return VINF_SUCCESS;
1903	}
1904
1905
1906	/**
1907	* Fetches the next opcode quad word, returns automatically on failure.
1908	*
1909	* @param a_pu64 Where to return the opcode quad word.
1910	* @remark Implicitly references pIemCpu.
1911	*/
1912	#define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1913	do \
1914	{ \
1915	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pIemCpu, (a_pu64)); \
1916	if (rcStrict2 != VINF_SUCCESS) \
1917	return rcStrict2; \
1918	} while (0)
1919
1920
1921	/** @name Misc Worker Functions.
1922	* @{
1923	*/
1924
1925
1926	/**
1927	* Validates a new SS segment.
1928	*
1929	* @returns VBox strict status code.
1930	* @param pIemCpu The IEM per CPU instance data.
1931	* @param pCtx The CPU context.
1932	* @param NewSS The new SS selctor.
1933	* @param uCpl The CPL to load the stack for.
1934	* @param pDesc Where to return the descriptor.
1935	*/
1936	static VBOXSTRICTRC iemMiscValidateNewSS(PIEMCPU pIemCpu, PCCPUMCTX pCtx, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
1937	{
1938	NOREF(pCtx);
1939
1940	/* Null selectors are not allowed (we're not called for dispatching
1941	interrupts with SS=0 in long mode). */
1942	if (!(NewSS & X86_SEL_MASK_OFF_RPL))
1943	{
1944	Log(("iemMiscValidateNewSSandRsp: #x - null selector -> #TS(0)\n", NewSS));
1945	return iemRaiseTaskSwitchFault0(pIemCpu);
1946	}
1947
1948	/** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
1949	if ((NewSS & X86_SEL_RPL) != uCpl)
1950	{
1951	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
1952	return iemRaiseTaskSwitchFaultBySelector(pIemCpu, NewSS);
1953	}
1954
1955	/*
1956	* Read the descriptor.
1957	*/
1958	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pIemCpu, pDesc, NewSS, X86_XCPT_TS);
1959	if (rcStrict != VINF_SUCCESS)
1960	return rcStrict;
1961
1962	/*
1963	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1964	*/
1965	if (!pDesc->Legacy.Gen.u1DescType)
1966	{
1967	Log(("iemMiscValidateNewSSandRsp: %#x - system selector -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1968	return iemRaiseTaskSwitchFaultBySelector(pIemCpu, NewSS);
1969	}
1970
1971	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1972	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1973	{
1974	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1975	return iemRaiseTaskSwitchFaultBySelector(pIemCpu, NewSS);
1976	}
1977	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1978	{
1979	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1980	return iemRaiseTaskSwitchFaultBySelector(pIemCpu, NewSS);
1981	}
1982
1983	/* Is it there? */
1984	/** @todo testcase: Is this checked before the canonical / limit check below? */
1985	if (!pDesc->Legacy.Gen.u1Present)
1986	{
1987	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1988	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewSS);
1989	}
1990
1991	return VINF_SUCCESS;
1992	}
1993
1994
1995	/**
1996	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1997	* not.
1998	*
1999	* @param a_pIemCpu The IEM per CPU data.
2000	* @param a_pCtx The CPU context.
2001	*/
2002	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2003	# define IEMMISC_GET_EFL(a_pIemCpu, a_pCtx) \
2004	( IEM_VERIFICATION_ENABLED(a_pIemCpu) \
2005	? (a_pCtx)->eflags.u \
2006	: CPUMRawGetEFlags(IEMCPU_TO_VMCPU(a_pIemCpu)) )
2007	#else
2008	# define IEMMISC_GET_EFL(a_pIemCpu, a_pCtx) \
2009	( (a_pCtx)->eflags.u )
2010	#endif
2011
2012	/**
2013	* Updates the EFLAGS in the correct manner wrt. PATM.
2014	*
2015	* @param a_pIemCpu The IEM per CPU data.
2016	* @param a_pCtx The CPU context.
2017	*/
2018	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2019	# define IEMMISC_SET_EFL(a_pIemCpu, a_pCtx, a_fEfl) \
2020	do { \
2021	if (IEM_VERIFICATION_ENABLED(a_pIemCpu)) \
2022	(a_pCtx)->eflags.u = (a_fEfl); \
2023	else \
2024	CPUMRawSetEFlags(IEMCPU_TO_VMCPU(a_pIemCpu), a_fEfl); \
2025	} while (0)
2026	#else
2027	# define IEMMISC_SET_EFL(a_pIemCpu, a_pCtx, a_fEfl) \
2028	do { \
2029	(a_pCtx)->eflags.u = (a_fEfl); \
2030	} while (0)
2031	#endif
2032
2033
2034	/** @} */
2035
2036	/** @name Raising Exceptions.
2037	*
2038	* @{
2039	*/
2040
2041	/** @name IEM_XCPT_FLAGS_XXX - flags for iemRaiseXcptOrInt.
2042	* @{ */
2043	/** CPU exception. */
2044	#define IEM_XCPT_FLAGS_T_CPU_XCPT RT_BIT_32(0)
2045	/** External interrupt (from PIC, APIC, whatever). */
2046	#define IEM_XCPT_FLAGS_T_EXT_INT RT_BIT_32(1)
2047	/** Software interrupt (int or into, not bound).
2048	* Returns to the following instruction */
2049	#define IEM_XCPT_FLAGS_T_SOFT_INT RT_BIT_32(2)
2050	/** Takes an error code. */
2051	#define IEM_XCPT_FLAGS_ERR RT_BIT_32(3)
2052	/** Takes a CR2. */
2053	#define IEM_XCPT_FLAGS_CR2 RT_BIT_32(4)
2054	/** Generated by the breakpoint instruction. */
2055	#define IEM_XCPT_FLAGS_BP_INSTR RT_BIT_32(5)
2056	/** Generated by a DRx instruction breakpoint and RF should be cleared. */
2057	#define IEM_XCPT_FLAGS_DRx_INSTR_BP RT_BIT_32(6)
2058	/** @} */
2059
2060
2061	/**
2062	* Loads the specified stack far pointer from the TSS.
2063	*
2064	* @returns VBox strict status code.
2065	* @param pIemCpu The IEM per CPU instance data.
2066	* @param pCtx The CPU context.
2067	* @param uCpl The CPL to load the stack for.
2068	* @param pSelSS Where to return the new stack segment.
2069	* @param puEsp Where to return the new stack pointer.
2070	*/
2071	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t uCpl,
2072	PRTSEL pSelSS, uint32_t *puEsp)
2073	{
2074	VBOXSTRICTRC rcStrict;
2075	Assert(uCpl < 4);
2076	puEsp = 0; / make gcc happy */
2077	pSelSS = 0; / make gcc happy */
2078
2079	switch (pCtx->tr.Attr.n.u4Type)
2080	{
2081	/*
2082	* 16-bit TSS (X86TSS16).
2083	*/
2084	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed();
2085	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
2086	{
2087	uint32_t off = uCpl * 4 + 2;
2088	if (off + 4 > pCtx->tr.u32Limit)
2089	{
2090	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->tr.u32Limit));
2091	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
2092	}
2093
2094	uint32_t u32Tmp = 0; /* gcc maybe... */
2095	rcStrict = iemMemFetchSysU32(pIemCpu, &u32Tmp, UINT8_MAX, pCtx->tr.u64Base + off);
2096	if (rcStrict == VINF_SUCCESS)
2097	{
2098	*puEsp = RT_LOWORD(u32Tmp);
2099	*pSelSS = RT_HIWORD(u32Tmp);
2100	return VINF_SUCCESS;
2101	}
2102	break;
2103	}
2104
2105	/*
2106	* 32-bit TSS (X86TSS32).
2107	*/
2108	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed();
2109	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
2110	{
2111	uint32_t off = uCpl * 8 + 4;
2112	if (off + 7 > pCtx->tr.u32Limit)
2113	{
2114	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->tr.u32Limit));
2115	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
2116	}
2117
2118	uint64_t u64Tmp;
2119	rcStrict = iemMemFetchSysU64(pIemCpu, &u64Tmp, UINT8_MAX, pCtx->tr.u64Base + off);
2120	if (rcStrict == VINF_SUCCESS)
2121	{
2122	*puEsp = u64Tmp & UINT32_MAX;
2123	*pSelSS = (RTSEL)(u64Tmp >> 32);
2124	return VINF_SUCCESS;
2125	}
2126	break;
2127	}
2128
2129	default:
2130	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
2131	}
2132	return rcStrict;
2133	}
2134
2135
2136	/**
2137	* Loads the specified stack pointer from the 64-bit TSS.
2138	*
2139	* @returns VBox strict status code.
2140	* @param pIemCpu The IEM per CPU instance data.
2141	* @param pCtx The CPU context.
2142	* @param uCpl The CPL to load the stack for.
2143	* @param uIst The interrupt stack table index, 0 if to use uCpl.
2144	* @param puRsp Where to return the new stack pointer.
2145	*/
2146	static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t uCpl, uint8_t uIst,
2147	uint64_t *puRsp)
2148	{
2149	Assert(uCpl < 4);
2150	Assert(uIst < 8);
2151	puRsp = 0; / make gcc happy */
2152
2153	AssertReturn(pCtx->tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_INTERNAL_ERROR_2);
2154
2155	uint32_t off;
2156	if (uIst)
2157	off = (uIst - 1) * sizeof(uint64_t) + RT_OFFSETOF(X86TSS64, ist1);
2158	else
2159	off = uCpl * sizeof(uint64_t) + RT_OFFSETOF(X86TSS64, rsp0);
2160	if (off + sizeof(uint64_t) > pCtx->tr.u32Limit)
2161	{
2162	Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pCtx->tr.u32Limit));
2163	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
2164	}
2165
2166	return iemMemFetchSysU64(pIemCpu, puRsp, UINT8_MAX, pCtx->tr.u64Base + off);
2167	}
2168
2169
2170	/**
2171	* Adjust the CPU state according to the exception being raised.
2172	*
2173	* @param pCtx The CPU context.
2174	* @param u8Vector The exception that has been raised.
2175	*/
2176	DECLINLINE(void) iemRaiseXcptAdjustState(PCPUMCTX pCtx, uint8_t u8Vector)
2177	{
2178	switch (u8Vector)
2179	{
2180	case X86_XCPT_DB:
2181	pCtx->dr[7] &= ~X86_DR7_GD;
2182	break;
2183	/** @todo Read the AMD and Intel exception reference... */
2184	}
2185	}
2186
2187
2188	/**
2189	* Implements exceptions and interrupts for real mode.
2190	*
2191	* @returns VBox strict status code.
2192	* @param pIemCpu The IEM per CPU instance data.
2193	* @param pCtx The CPU context.
2194	* @param cbInstr The number of bytes to offset rIP by in the return
2195	* address.
2196	* @param u8Vector The interrupt / exception vector number.
2197	* @param fFlags The flags.
2198	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2199	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2200	*/
2201	static VBOXSTRICTRC
2202	iemRaiseXcptOrIntInRealMode(PIEMCPU pIemCpu,
2203	PCPUMCTX pCtx,
2204	uint8_t cbInstr,
2205	uint8_t u8Vector,
2206	uint32_t fFlags,
2207	uint16_t uErr,
2208	uint64_t uCr2)
2209	{
2210	AssertReturn(pIemCpu->enmCpuMode == IEMMODE_16BIT, VERR_INTERNAL_ERROR_3);
2211	NOREF(uErr); NOREF(uCr2);
2212
2213	/*
2214	* Read the IDT entry.
2215	*/
2216	if (pCtx->idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
2217	{
2218	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
2219	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2220	}
2221	RTFAR16 Idte;
2222	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pIemCpu, (uint32_t *)&Idte, UINT8_MAX,
2223	pCtx->idtr.pIdt + UINT32_C(4) * u8Vector);
2224	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2225	return rcStrict;
2226
2227	/*
2228	* Push the stack frame.
2229	*/
2230	uint16_t *pu16Frame;
2231	uint64_t uNewRsp;
2232	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, 6, (void **)&pu16Frame, &uNewRsp);
2233	if (rcStrict != VINF_SUCCESS)
2234	return rcStrict;
2235
2236	uint32_t fEfl = IEMMISC_GET_EFL(pIemCpu, pCtx);
2237	pu16Frame[2] = (uint16_t)fEfl;
2238	pu16Frame[1] = (uint16_t)pCtx->cs.Sel;
2239	pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pCtx->ip + cbInstr : pCtx->ip;
2240	rcStrict = iemMemStackPushCommitSpecial(pIemCpu, pu16Frame, uNewRsp);
2241	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2242	return rcStrict;
2243
2244	/*
2245	* Load the vector address into cs:ip and make exception specific state
2246	* adjustments.
2247	*/
2248	pCtx->cs.Sel = Idte.sel;
2249	pCtx->cs.ValidSel = Idte.sel;
2250	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
2251	pCtx->cs.u64Base = (uint32_t)Idte.sel << 4;
2252	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
2253	pCtx->rip = Idte.off;
2254	fEfl &= ~X86_EFL_IF;
2255	IEMMISC_SET_EFL(pIemCpu, pCtx, fEfl);
2256
2257	/** @todo do we actually do this in real mode? */
2258	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
2259	iemRaiseXcptAdjustState(pCtx, u8Vector);
2260
2261	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2262	}
2263
2264
2265	/**
2266	* Loads a NULL data selector into when coming from V8086 mode.
2267	*
2268	* @param pIemCpu The IEM per CPU instance data.
2269	* @param pSReg Pointer to the segment register.
2270	*/
2271	static void iemHlpLoadNullDataSelectorOnV86Xcpt(PIEMCPU pIemCpu, PCPUMSELREG pSReg)
2272	{
2273	pSReg->Sel = 0;
2274	pSReg->ValidSel = 0;
2275	if (IEM_IS_GUEST_CPU_INTEL(pIemCpu) && !IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
2276	{
2277	/* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
2278	pSReg->Attr.u &= X86DESCATTR_DT \| X86DESCATTR_TYPE \| X86DESCATTR_DPL \| X86DESCATTR_G \| X86DESCATTR_D;
2279	pSReg->Attr.u \|= X86DESCATTR_UNUSABLE;
2280	}
2281	else
2282	{
2283	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2284	/** @todo check this on AMD-V */
2285	pSReg->u64Base = 0;
2286	pSReg->u32Limit = 0;
2287	}
2288	}
2289
2290
2291	/**
2292	* Loads a segment selector during a task switch in V8086 mode.
2293	*
2294	* @param pIemCpu The IEM per CPU instance data.
2295	* @param pSReg Pointer to the segment register.
2296	* @param uSel The selector value to load.
2297	*/
2298	static void iemHlpLoadSelectorInV86Mode(PIEMCPU pIemCpu, PCPUMSELREG pSReg, uint16_t uSel)
2299	{
2300	/* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
2301	pSReg->Sel = uSel;
2302	pSReg->ValidSel = uSel;
2303	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2304	pSReg->u64Base = uSel << 4;
2305	pSReg->u32Limit = 0xffff;
2306	pSReg->Attr.u = 0xf3;
2307	}
2308
2309
2310	/**
2311	* Loads a NULL data selector into a selector register, both the hidden and
2312	* visible parts, in protected mode.
2313	*
2314	* @param pIemCpu The IEM state of the calling EMT.
2315	* @param pSReg Pointer to the segment register.
2316	* @param uRpl The RPL.
2317	*/
2318	static void iemHlpLoadNullDataSelectorProt(PIEMCPU pIemCpu, PCPUMSELREG pSReg, RTSEL uRpl)
2319	{
2320	/** @todo Testcase: write a testcase checking what happends when loading a NULL
2321	* data selector in protected mode. */
2322	pSReg->Sel = uRpl;
2323	pSReg->ValidSel = uRpl;
2324	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2325	if (IEM_IS_GUEST_CPU_INTEL(pIemCpu) && !IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
2326	{
2327	/* VT-x (Intel 3960x) observed doing something like this. */
2328	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pIemCpu->uCpl << X86DESCATTR_DPL_SHIFT);
2329	pSReg->u32Limit = UINT32_MAX;
2330	pSReg->u64Base = 0;
2331	}
2332	else
2333	{
2334	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
2335	pSReg->u32Limit = 0;
2336	pSReg->u64Base = 0;
2337	}
2338	}
2339
2340
2341	/**
2342	* Loads a segment selector during a task switch in protected mode. In this task
2343	* switch scenario, we would throw #TS exceptions rather than #GPs.
2344	*
2345	* @returns VBox strict status code.
2346	* @param pIemCpu The IEM per CPU instance data.
2347	* @param pSReg Pointer to the segment register.
2348	* @param uSel The new selector value.
2349	*
2350	* @remarks This does -NOT- handle CS or SS.
2351	* @remarks This expects pIemCpu->uCpl to be up to date.
2352	*/
2353	static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PIEMCPU pIemCpu, PCPUMSELREG pSReg, uint16_t uSel)
2354	{
2355	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2356
2357	/* Null data selector. */
2358	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2359	{
2360	iemHlpLoadNullDataSelectorProt(pIemCpu, pSReg, uSel);
2361	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), pSReg));
2362	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_HIDDEN_SEL_REGS);
2363	return VINF_SUCCESS;
2364	}
2365
2366	/* Fetch the descriptor. */
2367	IEMSELDESC Desc;
2368	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pIemCpu, &Desc, uSel, X86_XCPT_TS);
2369	if (rcStrict != VINF_SUCCESS)
2370	{
2371	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
2372	VBOXSTRICTRC_VAL(rcStrict)));
2373	return rcStrict;
2374	}
2375
2376	/* Must be a data segment or readable code segment. */
2377	if ( !Desc.Legacy.Gen.u1DescType
2378	\|\| (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2379	{
2380	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
2381	Desc.Legacy.Gen.u4Type));
2382	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uSel & X86_SEL_MASK_OFF_RPL);
2383	}
2384
2385	/* Check privileges for data segments and non-conforming code segments. */
2386	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2387	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2388	{
2389	/* The RPL and the new CPL must be less than or equal to the DPL. */
2390	if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
2391	\|\| (pIemCpu->uCpl > Desc.Legacy.Gen.u2Dpl))
2392	{
2393	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
2394	uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
2395	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uSel & X86_SEL_MASK_OFF_RPL);
2396	}
2397	}
2398
2399	/* Is it there? */
2400	if (!Desc.Legacy.Gen.u1Present)
2401	{
2402	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
2403	return iemRaiseSelectorNotPresentWithErr(pIemCpu, uSel & X86_SEL_MASK_OFF_RPL);
2404	}
2405
2406	/* The base and limit. */
2407	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2408	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
2409
2410	/*
2411	* Ok, everything checked out fine. Now set the accessed bit before
2412	* committing the result into the registers.
2413	*/
2414	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2415	{
2416	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, uSel);
2417	if (rcStrict != VINF_SUCCESS)
2418	return rcStrict;
2419	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2420	}
2421
2422	/* Commit */
2423	pSReg->Sel = uSel;
2424	pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2425	pSReg->u32Limit = cbLimit;
2426	pSReg->u64Base = u64Base;
2427	pSReg->ValidSel = uSel;
2428	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2429	if (IEM_IS_GUEST_CPU_INTEL(pIemCpu) && !IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
2430	pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
2431
2432	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), pSReg));
2433	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_HIDDEN_SEL_REGS);
2434	return VINF_SUCCESS;
2435	}
2436
2437
2438	/**
2439	* Performs a task switch.
2440	*
2441	* If the task switch is the result of a JMP, CALL or IRET instruction, the
2442	* caller is responsible for performing the necessary checks (like DPL, TSS
2443	* present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
2444	* reference for JMP, CALL, IRET.
2445	*
2446	* If the task switch is the due to a software interrupt or hardware exception,
2447	* the caller is responsible for validating the TSS selector and descriptor. See
2448	* Intel Instruction reference for INT n.
2449	*
2450	* @returns VBox strict status code.
2451	* @param pIemCpu The IEM per CPU instance data.
2452	* @param pCtx The CPU context.
2453	* @param enmTaskSwitch What caused this task switch.
2454	* @param uNextEip The EIP effective after the task switch.
2455	* @param fFlags The flags.
2456	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2457	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2458	* @param SelTSS The TSS selector of the new task.
2459	* @param pNewDescTSS Pointer to the new TSS descriptor.
2460	*/
2461	static VBOXSTRICTRC iemTaskSwitch(PIEMCPU pIemCpu,
2462	PCPUMCTX pCtx,
2463	IEMTASKSWITCH enmTaskSwitch,
2464	uint32_t uNextEip,
2465	uint32_t fFlags,
2466	uint16_t uErr,
2467	uint64_t uCr2,
2468	RTSEL SelTSS,
2469	PIEMSELDESC pNewDescTSS)
2470	{
2471	Assert(!IEM_IS_REAL_MODE(pIemCpu));
2472	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2473
2474	uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
2475	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2476	\|\| uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2477	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2478	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2479
2480	bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2481	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2482
2483	Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RGv uNextEip=%#RGv\n", enmTaskSwitch, SelTSS,
2484	fIsNewTSS386, pCtx->eip, uNextEip));
2485
2486	/* Update CR2 in case it's a page-fault. */
2487	/** @todo This should probably be done much earlier in IEM/PGM. See
2488	* @bugref{5653} comment #49. */
2489	if (fFlags & IEM_XCPT_FLAGS_CR2)
2490	pCtx->cr2 = uCr2;
2491
2492	/*
2493	* Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
2494	* subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
2495	*/
2496	uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow \| (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
2497	uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
2498	if (uNewTSSLimit < uNewTSSLimitMin)
2499	{
2500	Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
2501	enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
2502	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2503	}
2504
2505	/*
2506	* Check the current TSS limit. The last written byte to the current TSS during the
2507	* task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
2508	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2509	*
2510	* The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
2511	* end up with smaller than "legal" TSS limits.
2512	*/
2513	uint32_t const uCurTSSLimit = pCtx->tr.u32Limit;
2514	uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
2515	if (uCurTSSLimit < uCurTSSLimitMin)
2516	{
2517	Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
2518	enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
2519	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2520	}
2521
2522	/*
2523	* Verify that the new TSS can be accessed and map it. Map only the required contents
2524	* and not the entire TSS.
2525	*/
2526	void *pvNewTSS;
2527	uint32_t cbNewTSS = uNewTSSLimitMin + 1;
2528	RTGCPTR GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
2529	AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, IntRedirBitmap) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
2530	/** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
2531	* not perform correct translation if this happens. See Intel spec. 7.2.1
2532	* "Task-State Segment" */
2533	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW);
2534	if (rcStrict != VINF_SUCCESS)
2535	{
2536	Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
2537	cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
2538	return rcStrict;
2539	}
2540
2541	/*
2542	* Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
2543	*/
2544	uint32_t u32EFlags = pCtx->eflags.u32;
2545	if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
2546	\|\| enmTaskSwitch == IEMTASKSWITCH_IRET)
2547	{
2548	PX86DESC pDescCurTSS;
2549	rcStrict = iemMemMap(pIemCpu, (void *)&pDescCurTSS, sizeof(pDescCurTSS), UINT8_MAX,
2550	pCtx->gdtr.pGdt + (pCtx->tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
2551	if (rcStrict != VINF_SUCCESS)
2552	{
2553	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2554	enmTaskSwitch, pCtx->gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2555	return rcStrict;
2556	}
2557
2558	pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2559	rcStrict = iemMemCommitAndUnmap(pIemCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
2560	if (rcStrict != VINF_SUCCESS)
2561	{
2562	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2563	enmTaskSwitch, pCtx->gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2564	return rcStrict;
2565	}
2566
2567	/* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
2568	if (enmTaskSwitch == IEMTASKSWITCH_IRET)
2569	{
2570	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2571	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2572	u32EFlags &= ~X86_EFL_NT;
2573	}
2574	}
2575
2576	/*
2577	* Save the CPU state into the current TSS.
2578	*/
2579	RTGCPTR GCPtrCurTSS = pCtx->tr.u64Base;
2580	if (GCPtrNewTSS == GCPtrCurTSS)
2581	{
2582	Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur\|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
2583	Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
2584	pCtx->cr3, pCtx->eip, pCtx->eflags.u32, pCtx->eax, pCtx->esp, pCtx->ebp, pCtx->cs.Sel, pCtx->ss.Sel, pCtx->ldtr.Sel));
2585	}
2586	if (fIsNewTSS386)
2587	{
2588	/*
2589	* Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
2590	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2591	*/
2592	void *pvCurTSS32;
2593	uint32_t offCurTSS = RT_OFFSETOF(X86TSS32, eip);
2594	uint32_t cbCurTSS = RT_OFFSETOF(X86TSS32, selLdt) - RT_OFFSETOF(X86TSS32, eip);
2595	AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
2596	rcStrict = iemMemMap(pIemCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
2597	if (rcStrict != VINF_SUCCESS)
2598	{
2599	Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2600	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2601	return rcStrict;
2602	}
2603
2604	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2605	PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
2606	pCurTSS32->eip = uNextEip;
2607	pCurTSS32->eflags = u32EFlags;
2608	pCurTSS32->eax = pCtx->eax;
2609	pCurTSS32->ecx = pCtx->ecx;
2610	pCurTSS32->edx = pCtx->edx;
2611	pCurTSS32->ebx = pCtx->ebx;
2612	pCurTSS32->esp = pCtx->esp;
2613	pCurTSS32->ebp = pCtx->ebp;
2614	pCurTSS32->esi = pCtx->esi;
2615	pCurTSS32->edi = pCtx->edi;
2616	pCurTSS32->es = pCtx->es.Sel;
2617	pCurTSS32->cs = pCtx->cs.Sel;
2618	pCurTSS32->ss = pCtx->ss.Sel;
2619	pCurTSS32->ds = pCtx->ds.Sel;
2620	pCurTSS32->fs = pCtx->fs.Sel;
2621	pCurTSS32->gs = pCtx->gs.Sel;
2622
2623	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
2624	if (rcStrict != VINF_SUCCESS)
2625	{
2626	Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2627	VBOXSTRICTRC_VAL(rcStrict)));
2628	return rcStrict;
2629	}
2630	}
2631	else
2632	{
2633	/*
2634	* Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
2635	*/
2636	void *pvCurTSS16;
2637	uint32_t offCurTSS = RT_OFFSETOF(X86TSS16, ip);
2638	uint32_t cbCurTSS = RT_OFFSETOF(X86TSS16, selLdt) - RT_OFFSETOF(X86TSS16, ip);
2639	AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
2640	rcStrict = iemMemMap(pIemCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
2641	if (rcStrict != VINF_SUCCESS)
2642	{
2643	Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2644	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2645	return rcStrict;
2646	}
2647
2648	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2649	PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
2650	pCurTSS16->ip = uNextEip;
2651	pCurTSS16->flags = u32EFlags;
2652	pCurTSS16->ax = pCtx->ax;
2653	pCurTSS16->cx = pCtx->cx;
2654	pCurTSS16->dx = pCtx->dx;
2655	pCurTSS16->bx = pCtx->bx;
2656	pCurTSS16->sp = pCtx->sp;
2657	pCurTSS16->bp = pCtx->bp;
2658	pCurTSS16->si = pCtx->si;
2659	pCurTSS16->di = pCtx->di;
2660	pCurTSS16->es = pCtx->es.Sel;
2661	pCurTSS16->cs = pCtx->cs.Sel;
2662	pCurTSS16->ss = pCtx->ss.Sel;
2663	pCurTSS16->ds = pCtx->ds.Sel;
2664
2665	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
2666	if (rcStrict != VINF_SUCCESS)
2667	{
2668	Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2669	VBOXSTRICTRC_VAL(rcStrict)));
2670	return rcStrict;
2671	}
2672	}
2673
2674	/*
2675	* Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
2676	*/
2677	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2678	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2679	{
2680	/* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
2681	PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
2682	pNewTSS->selPrev = pCtx->tr.Sel;
2683	}
2684
2685	/*
2686	* Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
2687	* it's done further below with error handling (e.g. CR3 changes will go through PGM).
2688	*/
2689	uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
2690	uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
2691	bool fNewDebugTrap;
2692	if (fIsNewTSS386)
2693	{
2694	PX86TSS32 pNewTSS32 = (PX86TSS32)pvNewTSS;
2695	uNewCr3 = (pCtx->cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
2696	uNewEip = pNewTSS32->eip;
2697	uNewEflags = pNewTSS32->eflags;
2698	uNewEax = pNewTSS32->eax;
2699	uNewEcx = pNewTSS32->ecx;
2700	uNewEdx = pNewTSS32->edx;
2701	uNewEbx = pNewTSS32->ebx;
2702	uNewEsp = pNewTSS32->esp;
2703	uNewEbp = pNewTSS32->ebp;
2704	uNewEsi = pNewTSS32->esi;
2705	uNewEdi = pNewTSS32->edi;
2706	uNewES = pNewTSS32->es;
2707	uNewCS = pNewTSS32->cs;
2708	uNewSS = pNewTSS32->ss;
2709	uNewDS = pNewTSS32->ds;
2710	uNewFS = pNewTSS32->fs;
2711	uNewGS = pNewTSS32->gs;
2712	uNewLdt = pNewTSS32->selLdt;
2713	fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
2714	}
2715	else
2716	{
2717	PX86TSS16 pNewTSS16 = (PX86TSS16)pvNewTSS;
2718	uNewCr3 = 0;
2719	uNewEip = pNewTSS16->ip;
2720	uNewEflags = pNewTSS16->flags;
2721	uNewEax = UINT32_C(0xffff0000) \| pNewTSS16->ax;
2722	uNewEcx = UINT32_C(0xffff0000) \| pNewTSS16->cx;
2723	uNewEdx = UINT32_C(0xffff0000) \| pNewTSS16->dx;
2724	uNewEbx = UINT32_C(0xffff0000) \| pNewTSS16->bx;
2725	uNewEsp = UINT32_C(0xffff0000) \| pNewTSS16->sp;
2726	uNewEbp = UINT32_C(0xffff0000) \| pNewTSS16->bp;
2727	uNewEsi = UINT32_C(0xffff0000) \| pNewTSS16->si;
2728	uNewEdi = UINT32_C(0xffff0000) \| pNewTSS16->di;
2729	uNewES = pNewTSS16->es;
2730	uNewCS = pNewTSS16->cs;
2731	uNewSS = pNewTSS16->ss;
2732	uNewDS = pNewTSS16->ds;
2733	uNewFS = 0;
2734	uNewGS = 0;
2735	uNewLdt = pNewTSS16->selLdt;
2736	fNewDebugTrap = false;
2737	}
2738
2739	if (GCPtrNewTSS == GCPtrCurTSS)
2740	Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
2741	uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
2742
2743	/*
2744	* We're done accessing the new TSS.
2745	*/
2746	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
2747	if (rcStrict != VINF_SUCCESS)
2748	{
2749	Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
2750	return rcStrict;
2751	}
2752
2753	/*
2754	* Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
2755	*/
2756	if (enmTaskSwitch != IEMTASKSWITCH_IRET)
2757	{
2758	rcStrict = iemMemMap(pIemCpu, (void *)&pNewDescTSS, sizeof(pNewDescTSS), UINT8_MAX,
2759	pCtx->gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
2760	if (rcStrict != VINF_SUCCESS)
2761	{
2762	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2763	enmTaskSwitch, pCtx->gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2764	return rcStrict;
2765	}
2766
2767	/* Check that the descriptor indicates the new TSS is available (not busy). */
2768	AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2769	\|\| pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
2770	("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
2771
2772	pNewDescTSS->Legacy.Gate.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2773	rcStrict = iemMemCommitAndUnmap(pIemCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
2774	if (rcStrict != VINF_SUCCESS)
2775	{
2776	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2777	enmTaskSwitch, pCtx->gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2778	return rcStrict;
2779	}
2780	}
2781
2782	/*
2783	* From this point on, we're technically in the new task. We will defer exceptions
2784	* until the completion of the task switch but before executing any instructions in the new task.
2785	*/
2786	pCtx->tr.Sel = SelTSS;
2787	pCtx->tr.ValidSel = SelTSS;
2788	pCtx->tr.fFlags = CPUMSELREG_FLAGS_VALID;
2789	pCtx->tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
2790	pCtx->tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
2791	pCtx->tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
2792	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_TR);
2793
2794	/* Set the busy bit in TR. */
2795	pCtx->tr.Attr.n.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2796	/* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
2797	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2798	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2799	{
2800	uNewEflags \|= X86_EFL_NT;
2801	}
2802
2803	pCtx->dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
2804	pCtx->cr0 \|= X86_CR0_TS;
2805	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_CR0);
2806
2807	pCtx->eip = uNewEip;
2808	pCtx->eax = uNewEax;
2809	pCtx->ecx = uNewEcx;
2810	pCtx->edx = uNewEdx;
2811	pCtx->ebx = uNewEbx;
2812	pCtx->esp = uNewEsp;
2813	pCtx->ebp = uNewEbp;
2814	pCtx->esi = uNewEsi;
2815	pCtx->edi = uNewEdi;
2816
2817	uNewEflags &= X86_EFL_LIVE_MASK;
2818	uNewEflags \|= X86_EFL_RA1_MASK;
2819	IEMMISC_SET_EFL(pIemCpu, pCtx, uNewEflags);
2820
2821	/*
2822	* Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
2823	* will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
2824	* due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
2825	*/
2826	pCtx->es.Sel = uNewES;
2827	pCtx->es.fFlags = CPUMSELREG_FLAGS_STALE;
2828	pCtx->es.Attr.u &= ~X86DESCATTR_P;
2829
2830	pCtx->cs.Sel = uNewCS;
2831	pCtx->cs.fFlags = CPUMSELREG_FLAGS_STALE;
2832	pCtx->cs.Attr.u &= ~X86DESCATTR_P;
2833
2834	pCtx->ss.Sel = uNewSS;
2835	pCtx->ss.fFlags = CPUMSELREG_FLAGS_STALE;
2836	pCtx->ss.Attr.u &= ~X86DESCATTR_P;
2837
2838	pCtx->ds.Sel = uNewDS;
2839	pCtx->ds.fFlags = CPUMSELREG_FLAGS_STALE;
2840	pCtx->ds.Attr.u &= ~X86DESCATTR_P;
2841
2842	pCtx->fs.Sel = uNewFS;
2843	pCtx->fs.fFlags = CPUMSELREG_FLAGS_STALE;
2844	pCtx->fs.Attr.u &= ~X86DESCATTR_P;
2845
2846	pCtx->gs.Sel = uNewGS;
2847	pCtx->gs.fFlags = CPUMSELREG_FLAGS_STALE;
2848	pCtx->gs.Attr.u &= ~X86DESCATTR_P;
2849	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_HIDDEN_SEL_REGS);
2850
2851	pCtx->ldtr.Sel = uNewLdt;
2852	pCtx->ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
2853	pCtx->ldtr.Attr.u &= ~X86DESCATTR_P;
2854	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_LDTR);
2855
2856	if (IEM_IS_GUEST_CPU_INTEL(pIemCpu) && !IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
2857	{
2858	pCtx->es.Attr.u \|= X86DESCATTR_UNUSABLE;
2859	pCtx->cs.Attr.u \|= X86DESCATTR_UNUSABLE;
2860	pCtx->ss.Attr.u \|= X86DESCATTR_UNUSABLE;
2861	pCtx->ds.Attr.u \|= X86DESCATTR_UNUSABLE;
2862	pCtx->fs.Attr.u \|= X86DESCATTR_UNUSABLE;
2863	pCtx->gs.Attr.u \|= X86DESCATTR_UNUSABLE;
2864	pCtx->ldtr.Attr.u \|= X86DESCATTR_UNUSABLE;
2865	}
2866
2867	/*
2868	* Switch CR3 for the new task.
2869	*/
2870	if ( fIsNewTSS386
2871	&& (pCtx->cr0 & X86_CR0_PG))
2872	{
2873	/** @todo Should we update and flush TLBs only if CR3 value actually changes? */
2874	if (!IEM_FULL_VERIFICATION_ENABLED(pIemCpu))
2875	{
2876	int rc = CPUMSetGuestCR3(IEMCPU_TO_VMCPU(pIemCpu), uNewCr3);
2877	AssertRCSuccessReturn(rc, rc);
2878	}
2879	else
2880	pCtx->cr3 = uNewCr3;
2881
2882	/* Inform PGM. */
2883	if (!IEM_FULL_VERIFICATION_ENABLED(pIemCpu))
2884	{
2885	int rc = PGMFlushTLB(IEMCPU_TO_VMCPU(pIemCpu), pCtx->cr3, !(pCtx->cr4 & X86_CR4_PGE));
2886	AssertRCReturn(rc, rc);
2887	/* ignore informational status codes */
2888	}
2889	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_CR3);
2890	}
2891
2892	/*
2893	* Switch LDTR for the new task.
2894	*/
2895	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
2896	iemHlpLoadNullDataSelectorProt(pIemCpu, &pCtx->ldtr, uNewLdt);
2897	else
2898	{
2899	Assert(!pCtx->ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
2900
2901	IEMSELDESC DescNewLdt;
2902	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
2903	if (rcStrict != VINF_SUCCESS)
2904	{
2905	Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
2906	uNewLdt, pCtx->gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
2907	return rcStrict;
2908	}
2909	if ( !DescNewLdt.Legacy.Gen.u1Present
2910	\|\| DescNewLdt.Legacy.Gen.u1DescType
2911	\|\| DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
2912	{
2913	Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
2914	uNewLdt, DescNewLdt.Legacy.u));
2915	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
2916	}
2917
2918	pCtx->ldtr.ValidSel = uNewLdt;
2919	pCtx->ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
2920	pCtx->ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
2921	pCtx->ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
2922	pCtx->ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
2923	if (IEM_IS_GUEST_CPU_INTEL(pIemCpu) && !IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
2924	pCtx->ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
2925	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), &pCtx->ldtr));
2926	}
2927
2928	IEMSELDESC DescSS;
2929	if (IEM_IS_V86_MODE(pIemCpu))
2930	{
2931	pIemCpu->uCpl = 3;
2932	iemHlpLoadSelectorInV86Mode(pIemCpu, &pCtx->es, uNewES);
2933	iemHlpLoadSelectorInV86Mode(pIemCpu, &pCtx->cs, uNewCS);
2934	iemHlpLoadSelectorInV86Mode(pIemCpu, &pCtx->ss, uNewSS);
2935	iemHlpLoadSelectorInV86Mode(pIemCpu, &pCtx->ds, uNewDS);
2936	iemHlpLoadSelectorInV86Mode(pIemCpu, &pCtx->fs, uNewFS);
2937	iemHlpLoadSelectorInV86Mode(pIemCpu, &pCtx->gs, uNewGS);
2938	}
2939	else
2940	{
2941	uint8_t uNewCpl = (uNewCS & X86_SEL_RPL);
2942
2943	/*
2944	* Load the stack segment for the new task.
2945	*/
2946	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
2947	{
2948	Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
2949	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2950	}
2951
2952	/* Fetch the descriptor. */
2953	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescSS, uNewSS, X86_XCPT_TS);
2954	if (rcStrict != VINF_SUCCESS)
2955	{
2956	Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
2957	VBOXSTRICTRC_VAL(rcStrict)));
2958	return rcStrict;
2959	}
2960
2961	/* SS must be a data segment and writable. */
2962	if ( !DescSS.Legacy.Gen.u1DescType
2963	\|\| (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2964	\|\| !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
2965	{
2966	Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
2967	uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
2968	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2969	}
2970
2971	/* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
2972	if ( (uNewSS & X86_SEL_RPL) != uNewCpl
2973	\|\| DescSS.Legacy.Gen.u2Dpl != uNewCpl)
2974	{
2975	Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
2976	uNewCpl));
2977	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2978	}
2979
2980	/* Is it there? */
2981	if (!DescSS.Legacy.Gen.u1Present)
2982	{
2983	Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
2984	return iemRaiseSelectorNotPresentWithErr(pIemCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2985	}
2986
2987	uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
2988	uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
2989
2990	/* Set the accessed bit before committing the result into SS. */
2991	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2992	{
2993	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, uNewSS);
2994	if (rcStrict != VINF_SUCCESS)
2995	return rcStrict;
2996	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2997	}
2998
2999	/* Commit SS. */
3000	pCtx->ss.Sel = uNewSS;
3001	pCtx->ss.ValidSel = uNewSS;
3002	pCtx->ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3003	pCtx->ss.u32Limit = cbLimit;
3004	pCtx->ss.u64Base = u64Base;
3005	pCtx->ss.fFlags = CPUMSELREG_FLAGS_VALID;
3006	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), &pCtx->ss));
3007
3008	/* CPL has changed, update IEM before loading rest of segments. */
3009	pIemCpu->uCpl = uNewCpl;
3010
3011	/*
3012	* Load the data segments for the new task.
3013	*/
3014	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pIemCpu, &pCtx->es, uNewES);
3015	if (rcStrict != VINF_SUCCESS)
3016	return rcStrict;
3017	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pIemCpu, &pCtx->ds, uNewDS);
3018	if (rcStrict != VINF_SUCCESS)
3019	return rcStrict;
3020	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pIemCpu, &pCtx->fs, uNewFS);
3021	if (rcStrict != VINF_SUCCESS)
3022	return rcStrict;
3023	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pIemCpu, &pCtx->gs, uNewGS);
3024	if (rcStrict != VINF_SUCCESS)
3025	return rcStrict;
3026
3027	/*
3028	* Load the code segment for the new task.
3029	*/
3030	if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
3031	{
3032	Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
3033	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3034	}
3035
3036	/* Fetch the descriptor. */
3037	IEMSELDESC DescCS;
3038	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescCS, uNewCS, X86_XCPT_TS);
3039	if (rcStrict != VINF_SUCCESS)
3040	{
3041	Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
3042	return rcStrict;
3043	}
3044
3045	/* CS must be a code segment. */
3046	if ( !DescCS.Legacy.Gen.u1DescType
3047	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3048	{
3049	Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
3050	DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
3051	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3052	}
3053
3054	/* For conforming CS, DPL must be less than or equal to the RPL. */
3055	if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
3056	&& DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
3057	{
3058	Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
3059	DescCS.Legacy.Gen.u2Dpl));
3060	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3061	}
3062
3063	/* For non-conforming CS, DPL must match RPL. */
3064	if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
3065	&& DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
3066	{
3067	Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
3068	DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
3069	return iemRaiseTaskSwitchFaultWithErr(pIemCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3070	}
3071
3072	/* Is it there? */
3073	if (!DescCS.Legacy.Gen.u1Present)
3074	{
3075	Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
3076	return iemRaiseSelectorNotPresentWithErr(pIemCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3077	}
3078
3079	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
3080	u64Base = X86DESC_BASE(&DescCS.Legacy);
3081
3082	/* Set the accessed bit before committing the result into CS. */
3083	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3084	{
3085	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, uNewCS);
3086	if (rcStrict != VINF_SUCCESS)
3087	return rcStrict;
3088	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3089	}
3090
3091	/* Commit CS. */
3092	pCtx->cs.Sel = uNewCS;
3093	pCtx->cs.ValidSel = uNewCS;
3094	pCtx->cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3095	pCtx->cs.u32Limit = cbLimit;
3096	pCtx->cs.u64Base = u64Base;
3097	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
3098	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), &pCtx->cs));
3099	}
3100
3101	/** @todo Debug trap. */
3102	if (fIsNewTSS386 && fNewDebugTrap)
3103	Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
3104
3105	/*
3106	* Construct the error code masks based on what caused this task switch.
3107	* See Intel Instruction reference for INT.
3108	*/
3109	uint16_t uExt;
3110	if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
3111	&& !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT))
3112	{
3113	uExt = 1;
3114	}
3115	else
3116	uExt = 0;
3117
3118	/*
3119	* Push any error code on to the new stack.
3120	*/
3121	if (fFlags & IEM_XCPT_FLAGS_ERR)
3122	{
3123	Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
3124	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3125	if (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN)
3126	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Expand down segments\n")); /** @todo Implement expand down segment support. */
3127
3128	/* Check that there is sufficient space on the stack. */
3129	uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
3130	if ( pCtx->esp - 1 > cbLimitSS
3131	\|\| pCtx->esp < cbStackFrame)
3132	{
3133	/** @todo Intel says #SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
3134	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n", pCtx->ss.Sel, pCtx->esp,
3135	cbStackFrame));
3136	return iemRaiseStackSelectorNotPresentWithErr(pIemCpu, uExt);
3137	}
3138
3139	if (fIsNewTSS386)
3140	rcStrict = iemMemStackPushU32(pIemCpu, uErr);
3141	else
3142	rcStrict = iemMemStackPushU16(pIemCpu, uErr);
3143	if (rcStrict != VINF_SUCCESS)
3144	{
3145	Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n", fIsNewTSS386 ? "32" : "16",
3146	VBOXSTRICTRC_VAL(rcStrict)));
3147	return rcStrict;
3148	}
3149	}
3150
3151	/* Check the new EIP against the new CS limit. */
3152	if (pCtx->eip > pCtx->cs.u32Limit)
3153	{
3154	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RGv CS limit=%u -> #GP(0)\n",
3155	pCtx->eip, pCtx->cs.u32Limit));
3156	/** @todo Intel says #GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
3157	return iemRaiseGeneralProtectionFault(pIemCpu, uExt);
3158	}
3159
3160	Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pCtx->cs.Sel, pCtx->eip, pCtx->ss.Sel));
3161	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3162	}
3163
3164
3165	/**
3166	* Implements exceptions and interrupts for protected mode.
3167	*
3168	* @returns VBox strict status code.
3169	* @param pIemCpu The IEM per CPU instance data.
3170	* @param pCtx The CPU context.
3171	* @param cbInstr The number of bytes to offset rIP by in the return
3172	* address.
3173	* @param u8Vector The interrupt / exception vector number.
3174	* @param fFlags The flags.
3175	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3176	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3177	*/
3178	static VBOXSTRICTRC
3179	iemRaiseXcptOrIntInProtMode(PIEMCPU pIemCpu,
3180	PCPUMCTX pCtx,
3181	uint8_t cbInstr,
3182	uint8_t u8Vector,
3183	uint32_t fFlags,
3184	uint16_t uErr,
3185	uint64_t uCr2)
3186	{
3187	NOREF(cbInstr);
3188
3189	/*
3190	* Read the IDT entry.
3191	*/
3192	if (pCtx->idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
3193	{
3194	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
3195	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3196	}
3197	X86DESC Idte;
3198	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &Idte.u, UINT8_MAX,
3199	pCtx->idtr.pIdt + UINT32_C(8) * u8Vector);
3200	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3201	return rcStrict;
3202	Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
3203	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3204	Idte.Gate.u4ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3205
3206	/*
3207	* Check the descriptor type, DPL and such.
3208	* ASSUMES this is done in the same order as described for call-gate calls.
3209	*/
3210	if (Idte.Gate.u1DescType)
3211	{
3212	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3213	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3214	}
3215	bool fTaskGate = false;
3216	uint8_t f32BitGate = true;
3217	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3218	switch (Idte.Gate.u4Type)
3219	{
3220	case X86_SEL_TYPE_SYS_UNDEFINED:
3221	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
3222	case X86_SEL_TYPE_SYS_LDT:
3223	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
3224	case X86_SEL_TYPE_SYS_286_CALL_GATE:
3225	case X86_SEL_TYPE_SYS_UNDEFINED2:
3226	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
3227	case X86_SEL_TYPE_SYS_UNDEFINED3:
3228	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
3229	case X86_SEL_TYPE_SYS_386_CALL_GATE:
3230	case X86_SEL_TYPE_SYS_UNDEFINED4:
3231	{
3232	/** @todo check what actually happens when the type is wrong...
3233	* esp. call gates. */
3234	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3235	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3236	}
3237
3238	case X86_SEL_TYPE_SYS_286_INT_GATE:
3239	f32BitGate = false;
3240	case X86_SEL_TYPE_SYS_386_INT_GATE:
3241	fEflToClear \|= X86_EFL_IF;
3242	break;
3243
3244	case X86_SEL_TYPE_SYS_TASK_GATE:
3245	fTaskGate = true;
3246	#ifndef IEM_IMPLEMENTS_TASKSWITCH
3247	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
3248	#endif
3249	break;
3250
3251	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
3252	f32BitGate = false;
3253	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
3254	break;
3255
3256	IEM_NOT_REACHED_DEFAULT_CASE_RET();
3257	}
3258
3259	/* Check DPL against CPL if applicable. */
3260	if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
3261	{
3262	if (pIemCpu->uCpl > Idte.Gate.u2Dpl)
3263	{
3264	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pIemCpu->uCpl, Idte.Gate.u2Dpl));
3265	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3266	}
3267	}
3268
3269	/* Is it there? */
3270	if (!Idte.Gate.u1Present)
3271	{
3272	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
3273	return iemRaiseSelectorNotPresentWithErr(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3274	}
3275
3276	/* Is it a task-gate? */
3277	if (fTaskGate)
3278	{
3279	/*
3280	* Construct the error code masks based on what caused this task switch.
3281	* See Intel Instruction reference for INT.
3282	*/
3283	uint16_t const uExt = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? 0 : 1;
3284	uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
3285	RTSEL SelTSS = Idte.Gate.u16Sel;
3286
3287	/*
3288	* Fetch the TSS descriptor in the GDT.
3289	*/
3290	IEMSELDESC DescTSS;
3291	rcStrict = iemMemFetchSelDescWithErr(pIemCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) \| uExt);
3292	if (rcStrict != VINF_SUCCESS)
3293	{
3294	Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
3295	VBOXSTRICTRC_VAL(rcStrict)));
3296	return rcStrict;
3297	}
3298
3299	/* The TSS descriptor must be a system segment and be available (not busy). */
3300	if ( DescTSS.Legacy.Gen.u1DescType
3301	\|\| ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
3302	&& DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
3303	{
3304	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
3305	u8Vector, SelTSS, DescTSS.Legacy.au64));
3306	return iemRaiseGeneralProtectionFault(pIemCpu, (SelTSS & uSelMask) \| uExt);
3307	}
3308
3309	/* The TSS must be present. */
3310	if (!DescTSS.Legacy.Gen.u1Present)
3311	{
3312	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
3313	return iemRaiseSelectorNotPresentWithErr(pIemCpu, (SelTSS & uSelMask) \| uExt);
3314	}
3315
3316	/* Do the actual task switch. */
3317	return iemTaskSwitch(pIemCpu, pCtx, IEMTASKSWITCH_INT_XCPT, pCtx->eip, fFlags, uErr, uCr2, SelTSS, &DescTSS);
3318	}
3319
3320	/* A null CS is bad. */
3321	RTSEL NewCS = Idte.Gate.u16Sel;
3322	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3323	{
3324	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3325	return iemRaiseGeneralProtectionFault0(pIemCpu);
3326	}
3327
3328	/* Fetch the descriptor for the new CS. */
3329	IEMSELDESC DescCS;
3330	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
3331	if (rcStrict != VINF_SUCCESS)
3332	{
3333	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3334	return rcStrict;
3335	}
3336
3337	/* Must be a code segment. */
3338	if (!DescCS.Legacy.Gen.u1DescType)
3339	{
3340	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3341	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3342	}
3343	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3344	{
3345	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3346	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3347	}
3348
3349	/* Don't allow lowering the privilege level. */
3350	/** @todo Does the lowering of privileges apply to software interrupts
3351	* only? This has bearings on the more-privileged or
3352	* same-privilege stack behavior further down. A testcase would
3353	* be nice. */
3354	if (DescCS.Legacy.Gen.u2Dpl > pIemCpu->uCpl)
3355	{
3356	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3357	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
3358	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3359	}
3360
3361	/* Make sure the selector is present. */
3362	if (!DescCS.Legacy.Gen.u1Present)
3363	{
3364	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3365	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewCS);
3366	}
3367
3368	/* Check the new EIP against the new CS limit. */
3369	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
3370	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
3371	? Idte.Gate.u16OffsetLow
3372	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
3373	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3374	if (uNewEip > cbLimitCS)
3375	{
3376	Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
3377	u8Vector, uNewEip, cbLimitCS, NewCS));
3378	return iemRaiseGeneralProtectionFault(pIemCpu, 0);
3379	}
3380
3381	/* Calc the flag image to push. */
3382	uint32_t fEfl = IEMMISC_GET_EFL(pIemCpu, pCtx);
3383	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3384	fEfl &= ~X86_EFL_RF;
3385	else if (!IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
3386	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3387
3388	/* From V8086 mode only go to CPL 0. */
3389	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3390	? pIemCpu->uCpl : DescCS.Legacy.Gen.u2Dpl;
3391	if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
3392	{
3393	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
3394	return iemRaiseGeneralProtectionFault(pIemCpu, 0);
3395	}
3396
3397	/*
3398	* If the privilege level changes, we need to get a new stack from the TSS.
3399	* This in turns means validating the new SS and ESP...
3400	*/
3401	if (uNewCpl != pIemCpu->uCpl)
3402	{
3403	RTSEL NewSS;
3404	uint32_t uNewEsp;
3405	rcStrict = iemRaiseLoadStackFromTss32Or16(pIemCpu, pCtx, uNewCpl, &NewSS, &uNewEsp);
3406	if (rcStrict != VINF_SUCCESS)
3407	return rcStrict;
3408
3409	IEMSELDESC DescSS;
3410	rcStrict = iemMiscValidateNewSS(pIemCpu, pCtx, NewSS, uNewCpl, &DescSS);
3411	if (rcStrict != VINF_SUCCESS)
3412	return rcStrict;
3413
3414	/* Check that there is sufficient space for the stack frame. */
3415	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3416	if (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN)
3417	{
3418	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Expand down segments\n")); /** @todo Implement expand down segment support. */
3419	}
3420
3421	uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
3422	? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
3423	: (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
3424	if ( uNewEsp - 1 > cbLimitSS
3425	\|\| uNewEsp < cbStackFrame)
3426	{
3427	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
3428	u8Vector, NewSS, uNewEsp, cbStackFrame));
3429	return iemRaiseSelectorBoundsBySelector(pIemCpu, NewSS);
3430	}
3431
3432	/*
3433	* Start making changes.
3434	*/
3435
3436	/* Create the stack frame. */
3437	RTPTRUNION uStackFrame;
3438	rcStrict = iemMemMap(pIemCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3439	uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy), IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
3440	if (rcStrict != VINF_SUCCESS)
3441	return rcStrict;
3442	void * const pvStackFrame = uStackFrame.pv;
3443	if (f32BitGate)
3444	{
3445	if (fFlags & IEM_XCPT_FLAGS_ERR)
3446	*uStackFrame.pu32++ = uErr;
3447	uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pCtx->eip + cbInstr : pCtx->eip;
3448	uStackFrame.pu32[1] = (pCtx->cs.Sel & ~X86_SEL_RPL) \| pIemCpu->uCpl;
3449	uStackFrame.pu32[2] = fEfl;
3450	uStackFrame.pu32[3] = pCtx->esp;
3451	uStackFrame.pu32[4] = pCtx->ss.Sel;
3452	if (fEfl & X86_EFL_VM)
3453	{
3454	uStackFrame.pu32[1] = pCtx->cs.Sel;
3455	uStackFrame.pu32[5] = pCtx->es.Sel;
3456	uStackFrame.pu32[6] = pCtx->ds.Sel;
3457	uStackFrame.pu32[7] = pCtx->fs.Sel;
3458	uStackFrame.pu32[8] = pCtx->gs.Sel;
3459	}
3460	}
3461	else
3462	{
3463	if (fFlags & IEM_XCPT_FLAGS_ERR)
3464	*uStackFrame.pu16++ = uErr;
3465	uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pCtx->ip + cbInstr : pCtx->ip;
3466	uStackFrame.pu16[1] = (pCtx->cs.Sel & ~X86_SEL_RPL) \| pIemCpu->uCpl;
3467	uStackFrame.pu16[2] = fEfl;
3468	uStackFrame.pu16[3] = pCtx->sp;
3469	uStackFrame.pu16[4] = pCtx->ss.Sel;
3470	if (fEfl & X86_EFL_VM)
3471	{
3472	uStackFrame.pu16[1] = pCtx->cs.Sel;
3473	uStackFrame.pu16[5] = pCtx->es.Sel;
3474	uStackFrame.pu16[6] = pCtx->ds.Sel;
3475	uStackFrame.pu16[7] = pCtx->fs.Sel;
3476	uStackFrame.pu16[8] = pCtx->gs.Sel;
3477	}
3478	}
3479	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3480	if (rcStrict != VINF_SUCCESS)
3481	return rcStrict;
3482
3483	/* Mark the selectors 'accessed' (hope this is the correct time). */
3484	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3485	* after pushing the stack frame? (Write protect the gdt + stack to
3486	* find out.) */
3487	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3488	{
3489	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
3490	if (rcStrict != VINF_SUCCESS)
3491	return rcStrict;
3492	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3493	}
3494
3495	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3496	{
3497	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewSS);
3498	if (rcStrict != VINF_SUCCESS)
3499	return rcStrict;
3500	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3501	}
3502
3503	/*
3504	* Start comitting the register changes (joins with the DPL=CPL branch).
3505	*/
3506	pCtx->ss.Sel = NewSS;
3507	pCtx->ss.ValidSel = NewSS;
3508	pCtx->ss.fFlags = CPUMSELREG_FLAGS_VALID;
3509	pCtx->ss.u32Limit = cbLimitSS;
3510	pCtx->ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3511	pCtx->ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3512	pCtx->rsp = uNewEsp - cbStackFrame; /** @todo Is the high word cleared for 16-bit stacks and/or interrupt handlers? */
3513	pIemCpu->uCpl = uNewCpl;
3514
3515	if (fEfl & X86_EFL_VM)
3516	{
3517	iemHlpLoadNullDataSelectorOnV86Xcpt(pIemCpu, &pCtx->gs);
3518	iemHlpLoadNullDataSelectorOnV86Xcpt(pIemCpu, &pCtx->fs);
3519	iemHlpLoadNullDataSelectorOnV86Xcpt(pIemCpu, &pCtx->es);
3520	iemHlpLoadNullDataSelectorOnV86Xcpt(pIemCpu, &pCtx->ds);
3521	}
3522	}
3523	/*
3524	* Same privilege, no stack change and smaller stack frame.
3525	*/
3526	else
3527	{
3528	uint64_t uNewRsp;
3529	RTPTRUNION uStackFrame;
3530	uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
3531	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
3532	if (rcStrict != VINF_SUCCESS)
3533	return rcStrict;
3534	void * const pvStackFrame = uStackFrame.pv;
3535
3536	if (f32BitGate)
3537	{
3538	if (fFlags & IEM_XCPT_FLAGS_ERR)
3539	*uStackFrame.pu32++ = uErr;
3540	uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pCtx->eip + cbInstr : pCtx->eip;
3541	uStackFrame.pu32[1] = (pCtx->cs.Sel & ~X86_SEL_RPL) \| pIemCpu->uCpl;
3542	uStackFrame.pu32[2] = fEfl;
3543	}
3544	else
3545	{
3546	if (fFlags & IEM_XCPT_FLAGS_ERR)
3547	*uStackFrame.pu16++ = uErr;
3548	uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pCtx->eip + cbInstr : pCtx->eip;
3549	uStackFrame.pu16[1] = (pCtx->cs.Sel & ~X86_SEL_RPL) \| pIemCpu->uCpl;
3550	uStackFrame.pu16[2] = fEfl;
3551	}
3552	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
3553	if (rcStrict != VINF_SUCCESS)
3554	return rcStrict;
3555
3556	/* Mark the CS selector as 'accessed'. */
3557	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3558	{
3559	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
3560	if (rcStrict != VINF_SUCCESS)
3561	return rcStrict;
3562	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3563	}
3564
3565	/*
3566	* Start committing the register changes (joins with the other branch).
3567	*/
3568	pCtx->rsp = uNewRsp;
3569	}
3570
3571	/* ... register committing continues. */
3572	pCtx->cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3573	pCtx->cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3574	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
3575	pCtx->cs.u32Limit = cbLimitCS;
3576	pCtx->cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3577	pCtx->cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3578
3579	pCtx->rip = uNewEip;
3580	fEfl &= ~fEflToClear;
3581	IEMMISC_SET_EFL(pIemCpu, pCtx, fEfl);
3582
3583	if (fFlags & IEM_XCPT_FLAGS_CR2)
3584	pCtx->cr2 = uCr2;
3585
3586	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3587	iemRaiseXcptAdjustState(pCtx, u8Vector);
3588
3589	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3590	}
3591
3592
3593	/**
3594	* Implements exceptions and interrupts for long mode.
3595	*
3596	* @returns VBox strict status code.
3597	* @param pIemCpu The IEM per CPU instance data.
3598	* @param pCtx The CPU context.
3599	* @param cbInstr The number of bytes to offset rIP by in the return
3600	* address.
3601	* @param u8Vector The interrupt / exception vector number.
3602	* @param fFlags The flags.
3603	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3604	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3605	*/
3606	static VBOXSTRICTRC
3607	iemRaiseXcptOrIntInLongMode(PIEMCPU pIemCpu,
3608	PCPUMCTX pCtx,
3609	uint8_t cbInstr,
3610	uint8_t u8Vector,
3611	uint32_t fFlags,
3612	uint16_t uErr,
3613	uint64_t uCr2)
3614	{
3615	NOREF(cbInstr);
3616
3617	/*
3618	* Read the IDT entry.
3619	*/
3620	uint16_t offIdt = (uint16_t)u8Vector << 4;
3621	if (pCtx->idtr.cbIdt < offIdt + 7)
3622	{
3623	Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
3624	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3625	}
3626	X86DESC64 Idte;
3627	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &Idte.au64[0], UINT8_MAX, pCtx->idtr.pIdt + offIdt);
3628	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3629	rcStrict = iemMemFetchSysU64(pIemCpu, &Idte.au64[1], UINT8_MAX, pCtx->idtr.pIdt + offIdt + 8);
3630	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3631	return rcStrict;
3632	Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
3633	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3634	Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3635
3636	/*
3637	* Check the descriptor type, DPL and such.
3638	* ASSUMES this is done in the same order as described for call-gate calls.
3639	*/
3640	if (Idte.Gate.u1DescType)
3641	{
3642	Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3643	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3644	}
3645	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3646	switch (Idte.Gate.u4Type)
3647	{
3648	case AMD64_SEL_TYPE_SYS_INT_GATE:
3649	fEflToClear \|= X86_EFL_IF;
3650	break;
3651	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
3652	break;
3653
3654	default:
3655	Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3656	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3657	}
3658
3659	/* Check DPL against CPL if applicable. */
3660	if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
3661	{
3662	if (pIemCpu->uCpl > Idte.Gate.u2Dpl)
3663	{
3664	Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pIemCpu->uCpl, Idte.Gate.u2Dpl));
3665	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3666	}
3667	}
3668
3669	/* Is it there? */
3670	if (!Idte.Gate.u1Present)
3671	{
3672	Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
3673	return iemRaiseSelectorNotPresentWithErr(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3674	}
3675
3676	/* A null CS is bad. */
3677	RTSEL NewCS = Idte.Gate.u16Sel;
3678	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3679	{
3680	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3681	return iemRaiseGeneralProtectionFault0(pIemCpu);
3682	}
3683
3684	/* Fetch the descriptor for the new CS. */
3685	IEMSELDESC DescCS;
3686	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescCS, NewCS, X86_XCPT_GP);
3687	if (rcStrict != VINF_SUCCESS)
3688	{
3689	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3690	return rcStrict;
3691	}
3692
3693	/* Must be a 64-bit code segment. */
3694	if (!DescCS.Long.Gen.u1DescType)
3695	{
3696	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3697	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3698	}
3699	if ( !DescCS.Long.Gen.u1Long
3700	\|\| DescCS.Long.Gen.u1DefBig
3701	\|\| !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
3702	{
3703	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
3704	u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
3705	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3706	}
3707
3708	/* Don't allow lowering the privilege level. For non-conforming CS
3709	selectors, the CS.DPL sets the privilege level the trap/interrupt
3710	handler runs at. For conforming CS selectors, the CPL remains
3711	unchanged, but the CS.DPL must be <= CPL. */
3712	/** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
3713	* when CPU in Ring-0. Result \#GP? */
3714	if (DescCS.Legacy.Gen.u2Dpl > pIemCpu->uCpl)
3715	{
3716	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3717	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
3718	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3719	}
3720
3721
3722	/* Make sure the selector is present. */
3723	if (!DescCS.Legacy.Gen.u1Present)
3724	{
3725	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3726	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewCS);
3727	}
3728
3729	/* Check that the new RIP is canonical. */
3730	uint64_t const uNewRip = Idte.Gate.u16OffsetLow
3731	\| ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
3732	\| ((uint64_t)Idte.Gate.u32OffsetTop << 32);
3733	if (!IEM_IS_CANONICAL(uNewRip))
3734	{
3735	Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
3736	return iemRaiseGeneralProtectionFault0(pIemCpu);
3737	}
3738
3739	/*
3740	* If the privilege level changes or if the IST isn't zero, we need to get
3741	* a new stack from the TSS.
3742	*/
3743	uint64_t uNewRsp;
3744	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3745	? pIemCpu->uCpl : DescCS.Legacy.Gen.u2Dpl;
3746	if ( uNewCpl != pIemCpu->uCpl
3747	\|\| Idte.Gate.u3IST != 0)
3748	{
3749	rcStrict = iemRaiseLoadStackFromTss64(pIemCpu, pCtx, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
3750	if (rcStrict != VINF_SUCCESS)
3751	return rcStrict;
3752	}
3753	else
3754	uNewRsp = pCtx->rsp;
3755	uNewRsp &= ~(uint64_t)0xf;
3756
3757	/*
3758	* Calc the flag image to push.
3759	*/
3760	uint32_t fEfl = IEMMISC_GET_EFL(pIemCpu, pCtx);
3761	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3762	fEfl &= ~X86_EFL_RF;
3763	else if (!IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
3764	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3765
3766	/*
3767	* Start making changes.
3768	*/
3769
3770	/* Create the stack frame. */
3771	uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
3772	RTPTRUNION uStackFrame;
3773	rcStrict = iemMemMap(pIemCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3774	uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
3775	if (rcStrict != VINF_SUCCESS)
3776	return rcStrict;
3777	void * const pvStackFrame = uStackFrame.pv;
3778
3779	if (fFlags & IEM_XCPT_FLAGS_ERR)
3780	*uStackFrame.pu64++ = uErr;
3781	uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pCtx->rip + cbInstr : pCtx->rip;
3782	uStackFrame.pu64[1] = (pCtx->cs.Sel & ~X86_SEL_RPL) \| pIemCpu->uCpl; /* CPL paranoia */
3783	uStackFrame.pu64[2] = fEfl;
3784	uStackFrame.pu64[3] = pCtx->rsp;
3785	uStackFrame.pu64[4] = pCtx->ss.Sel;
3786	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3787	if (rcStrict != VINF_SUCCESS)
3788	return rcStrict;
3789
3790	/* Mark the CS selectors 'accessed' (hope this is the correct time). */
3791	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3792	* after pushing the stack frame? (Write protect the gdt + stack to
3793	* find out.) */
3794	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3795	{
3796	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
3797	if (rcStrict != VINF_SUCCESS)
3798	return rcStrict;
3799	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3800	}
3801
3802	/*
3803	* Start comitting the register changes.
3804	*/
3805	/** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
3806	* hidden registers when interrupting 32-bit or 16-bit code! */
3807	if (uNewCpl != pIemCpu->uCpl)
3808	{
3809	pCtx->ss.Sel = 0 \| uNewCpl;
3810	pCtx->ss.ValidSel = 0 \| uNewCpl;
3811	pCtx->ss.fFlags = CPUMSELREG_FLAGS_VALID;
3812	pCtx->ss.u32Limit = UINT32_MAX;
3813	pCtx->ss.u64Base = 0;
3814	pCtx->ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) \| X86DESCATTR_UNUSABLE;
3815	}
3816	pCtx->rsp = uNewRsp - cbStackFrame;
3817	pCtx->cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3818	pCtx->cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3819	pCtx->cs.fFlags = CPUMSELREG_FLAGS_VALID;
3820	pCtx->cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
3821	pCtx->cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3822	pCtx->cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3823	pCtx->rip = uNewRip;
3824	pIemCpu->uCpl = uNewCpl;
3825
3826	fEfl &= ~fEflToClear;
3827	IEMMISC_SET_EFL(pIemCpu, pCtx, fEfl);
3828
3829	if (fFlags & IEM_XCPT_FLAGS_CR2)
3830	pCtx->cr2 = uCr2;
3831
3832	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3833	iemRaiseXcptAdjustState(pCtx, u8Vector);
3834
3835	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3836	}
3837
3838
3839	/**
3840	* Implements exceptions and interrupts.
3841	*
3842	* All exceptions and interrupts goes thru this function!
3843	*
3844	* @returns VBox strict status code.
3845	* @param pIemCpu The IEM per CPU instance data.
3846	* @param cbInstr The number of bytes to offset rIP by in the return
3847	* address.
3848	* @param u8Vector The interrupt / exception vector number.
3849	* @param fFlags The flags.
3850	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3851	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3852	*/
3853	DECL_NO_INLINE(static, VBOXSTRICTRC)
3854	iemRaiseXcptOrInt(PIEMCPU pIemCpu,
3855	uint8_t cbInstr,
3856	uint8_t u8Vector,
3857	uint32_t fFlags,
3858	uint16_t uErr,
3859	uint64_t uCr2)
3860	{
3861	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3862
3863	/*
3864	* Perform the V8086 IOPL check and upgrade the fault without nesting.
3865	*/
3866	if ( pCtx->eflags.Bits.u1VM
3867	&& pCtx->eflags.Bits.u2IOPL != 3
3868	&& (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
3869	&& (pCtx->cr0 & X86_CR0_PE) )
3870	{
3871	Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
3872	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3873	u8Vector = X86_XCPT_GP;
3874	uErr = 0;
3875	}
3876	#ifdef DBGFTRACE_ENABLED
3877	RTTraceBufAddMsgF(IEMCPU_TO_VM(pIemCpu)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
3878	pIemCpu->cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
3879	pCtx->cs.Sel, pCtx->rip, pCtx->ss.Sel, pCtx->rsp);
3880	#endif
3881
3882	/*
3883	* Do recursion accounting.
3884	*/
3885	uint8_t const uPrevXcpt = pIemCpu->uCurXcpt;
3886	uint32_t const fPrevXcpt = pIemCpu->fCurXcpt;
3887	if (pIemCpu->cXcptRecursions == 0)
3888	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
3889	u8Vector, pCtx->cs.Sel, pCtx->rip, cbInstr, fFlags, uErr, uCr2));
3890	else
3891	{
3892	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
3893	u8Vector, pCtx->cs.Sel, pCtx->rip, cbInstr, fFlags, uErr, uCr2, pIemCpu->uCurXcpt, pIemCpu->cXcptRecursions + 1, fPrevXcpt));
3894
3895	/** @todo double and tripple faults. */
3896	if (pIemCpu->cXcptRecursions >= 3)
3897	{
3898	#ifdef DEBUG_bird
3899	AssertFailed();
3900	#endif
3901	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
3902	}
3903
3904	/** @todo set X86_TRAP_ERR_EXTERNAL when appropriate.
3905	if (fPrevXcpt & IEM_XCPT_FLAGS_T_EXT_INT)
3906	{
3907	....
3908	} */
3909	}
3910	pIemCpu->cXcptRecursions++;
3911	pIemCpu->uCurXcpt = u8Vector;
3912	pIemCpu->fCurXcpt = fFlags;
3913
3914	/*
3915	* Extensive logging.
3916	*/
3917	#if defined(LOG_ENABLED) && defined(IN_RING3)
3918	if (LogIs3Enabled())
3919	{
3920	PVM pVM = IEMCPU_TO_VM(pIemCpu);
3921	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
3922	char szRegs[4096];
3923	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
3924	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
3925	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
3926	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
3927	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
3928	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
3929	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
3930	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
3931	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
3932	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
3933	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
3934	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
3935	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
3936	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
3937	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
3938	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
3939	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
3940	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
3941	" efer=%016VR{efer}\n"
3942	" pat=%016VR{pat}\n"
3943	" sf_mask=%016VR{sf_mask}\n"
3944	"krnl_gs_base=%016VR{krnl_gs_base}\n"
3945	" lstar=%016VR{lstar}\n"
3946	" star=%016VR{star} cstar=%016VR{cstar}\n"
3947	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
3948	);
3949
3950	char szInstr[256];
3951	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
3952	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
3953	szInstr, sizeof(szInstr), NULL);
3954	Log3(("%s%s\n", szRegs, szInstr));
3955	}
3956	#endif /* LOG_ENABLED */
3957
3958	/*
3959	* Call the mode specific worker function.
3960	*/
3961	VBOXSTRICTRC rcStrict;
3962	if (!(pCtx->cr0 & X86_CR0_PE))
3963	rcStrict = iemRaiseXcptOrIntInRealMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
3964	else if (pCtx->msrEFER & MSR_K6_EFER_LMA)
3965	rcStrict = iemRaiseXcptOrIntInLongMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
3966	else
3967	rcStrict = iemRaiseXcptOrIntInProtMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
3968
3969	/*
3970	* Unwind.
3971	*/
3972	pIemCpu->cXcptRecursions--;
3973	pIemCpu->uCurXcpt = uPrevXcpt;
3974	pIemCpu->fCurXcpt = fPrevXcpt;
3975	Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u\n",
3976	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pCtx->cs.Sel, pCtx->rip, pCtx->ss.Sel, pCtx->esp, pIemCpu->uCpl));
3977	return rcStrict;
3978	}
3979
3980
3981	/** \#DE - 00. */
3982	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDivideError(PIEMCPU pIemCpu)
3983	{
3984	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3985	}
3986
3987
3988	/** \#DB - 01.
3989	* @note This automatically clear DR7.GD. */
3990	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDebugException(PIEMCPU pIemCpu)
3991	{
3992	/** @todo set/clear RF. */
3993	pIemCpu->CTX_SUFF(pCtx)->dr[7] &= ~X86_DR7_GD;
3994	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3995	}
3996
3997
3998	/** \#UD - 06. */
3999	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PIEMCPU pIemCpu)
4000	{
4001	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4002	}
4003
4004
4005	/** \#NM - 07. */
4006	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PIEMCPU pIemCpu)
4007	{
4008	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4009	}
4010
4011
4012	/** \#TS(err) - 0a. */
4013	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PIEMCPU pIemCpu, uint16_t uErr)
4014	{
4015	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4016	}
4017
4018
4019	/** \#TS(tr) - 0a. */
4020	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu)
4021	{
4022	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4023	pIemCpu->CTX_SUFF(pCtx)->tr.Sel, 0);
4024	}
4025
4026
4027	/** \#TS(0) - 0a. */
4028	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFault0(PIEMCPU pIemCpu)
4029	{
4030	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4031	0, 0);
4032	}
4033
4034
4035	/** \#TS(err) - 0a. */
4036	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultBySelector(PIEMCPU pIemCpu, uint16_t uSel)
4037	{
4038	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4039	uSel & X86_SEL_MASK_OFF_RPL, 0);
4040	}
4041
4042
4043	/** \#NP(err) - 0b. */
4044	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr)
4045	{
4046	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4047	}
4048
4049
4050	/** \#NP(seg) - 0b. */
4051	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySegReg(PIEMCPU pIemCpu, uint32_t iSegReg)
4052	{
4053	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4054	iemSRegFetchU16(pIemCpu, iSegReg) & ~X86_SEL_RPL, 0);
4055	}
4056
4057
4058	/** \#NP(sel) - 0b. */
4059	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
4060	{
4061	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4062	uSel & ~X86_SEL_RPL, 0);
4063	}
4064
4065
4066	/** \#SS(seg) - 0c. */
4067	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
4068	{
4069	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4070	uSel & ~X86_SEL_RPL, 0);
4071	}
4072
4073
4074	/** \#SS(err) - 0c. */
4075	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr)
4076	{
4077	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4078	}
4079
4080
4081	/** \#GP(n) - 0d. */
4082	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr)
4083	{
4084	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4085	}
4086
4087
4088	/** \#GP(0) - 0d. */
4089	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu)
4090	{
4091	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4092	}
4093
4094
4095	/** \#GP(sel) - 0d. */
4096	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL Sel)
4097	{
4098	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4099	Sel & ~X86_SEL_RPL, 0);
4100	}
4101
4102
4103	/** \#GP(0) - 0d. */
4104	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseNotCanonical(PIEMCPU pIemCpu)
4105	{
4106	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4107	}
4108
4109
4110	/** \#GP(sel) - 0d. */
4111	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
4112	{
4113	NOREF(iSegReg); NOREF(fAccess);
4114	return iemRaiseXcptOrInt(pIemCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4115	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4116	}
4117
4118
4119	/** \#GP(sel) - 0d. */
4120	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel)
4121	{
4122	NOREF(Sel);
4123	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4124	}
4125
4126
4127	/** \#GP(sel) - 0d. */
4128	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
4129	{
4130	NOREF(iSegReg); NOREF(fAccess);
4131	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4132	}
4133
4134
4135	/** \#PF(n) - 0e. */
4136	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
4137	{
4138	uint16_t uErr;
4139	switch (rc)
4140	{
4141	case VERR_PAGE_NOT_PRESENT:
4142	case VERR_PAGE_TABLE_NOT_PRESENT:
4143	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
4144	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
4145	uErr = 0;
4146	break;
4147
4148	default:
4149	AssertMsgFailed(("%Rrc\n", rc));
4150	case VERR_ACCESS_DENIED:
4151	uErr = X86_TRAP_PF_P;
4152	break;
4153
4154	/** @todo reserved */
4155	}
4156
4157	if (pIemCpu->uCpl == 3)
4158	uErr \|= X86_TRAP_PF_US;
4159
4160	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
4161	&& ( (pIemCpu->CTX_SUFF(pCtx)->cr4 & X86_CR4_PAE)
4162	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) ) )
4163	uErr \|= X86_TRAP_PF_ID;
4164
4165	#if 0 /* This is so much non-sense, really. Why was it done like that? */
4166	/* Note! RW access callers reporting a WRITE protection fault, will clear
4167	the READ flag before calling. So, read-modify-write accesses (RW)
4168	can safely be reported as READ faults. */
4169	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
4170	uErr \|= X86_TRAP_PF_RW;
4171	#else
4172	if (fAccess & IEM_ACCESS_TYPE_WRITE)
4173	{
4174	if (!IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu) \|\| !(fAccess & IEM_ACCESS_TYPE_READ))
4175	uErr \|= X86_TRAP_PF_RW;
4176	}
4177	#endif
4178
4179	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
4180	uErr, GCPtrWhere);
4181	}
4182
4183
4184	/** \#MF(0) - 10. */
4185	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseMathFault(PIEMCPU pIemCpu)
4186	{
4187	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4188	}
4189
4190
4191	/** \#AC(0) - 11. */
4192	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PIEMCPU pIemCpu)
4193	{
4194	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4195	}
4196
4197
4198	/**
4199	* Macro for calling iemCImplRaiseDivideError().
4200	*
4201	* This enables us to add/remove arguments and force different levels of
4202	* inlining as we wish.
4203	*
4204	* @return Strict VBox status code.
4205	*/
4206	#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
4207	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
4208	{
4209	NOREF(cbInstr);
4210	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4211	}
4212
4213
4214	/**
4215	* Macro for calling iemCImplRaiseInvalidLockPrefix().
4216	*
4217	* This enables us to add/remove arguments and force different levels of
4218	* inlining as we wish.
4219	*
4220	* @return Strict VBox status code.
4221	*/
4222	#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
4223	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
4224	{
4225	NOREF(cbInstr);
4226	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4227	}
4228
4229
4230	/**
4231	* Macro for calling iemCImplRaiseInvalidOpcode().
4232	*
4233	* This enables us to add/remove arguments and force different levels of
4234	* inlining as we wish.
4235	*
4236	* @return Strict VBox status code.
4237	*/
4238	#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
4239	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
4240	{
4241	NOREF(cbInstr);
4242	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4243	}
4244
4245
4246	/** @} */
4247
4248
4249	/*
4250	*
4251	* Helpers routines.
4252	* Helpers routines.
4253	* Helpers routines.
4254	*
4255	*/
4256
4257	/**
4258	* Recalculates the effective operand size.
4259	*
4260	* @param pIemCpu The IEM state.
4261	*/
4262	static void iemRecalEffOpSize(PIEMCPU pIemCpu)
4263	{
4264	switch (pIemCpu->enmCpuMode)
4265	{
4266	case IEMMODE_16BIT:
4267	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
4268	break;
4269	case IEMMODE_32BIT:
4270	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
4271	break;
4272	case IEMMODE_64BIT:
4273	switch (pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
4274	{
4275	case 0:
4276	pIemCpu->enmEffOpSize = pIemCpu->enmDefOpSize;
4277	break;
4278	case IEM_OP_PRF_SIZE_OP:
4279	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
4280	break;
4281	case IEM_OP_PRF_SIZE_REX_W:
4282	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
4283	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
4284	break;
4285	}
4286	break;
4287	default:
4288	AssertFailed();
4289	}
4290	}
4291
4292
4293	/**
4294	* Sets the default operand size to 64-bit and recalculates the effective
4295	* operand size.
4296	*
4297	* @param pIemCpu The IEM state.
4298	*/
4299	static void iemRecalEffOpSize64Default(PIEMCPU pIemCpu)
4300	{
4301	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
4302	pIemCpu->enmDefOpSize = IEMMODE_64BIT;
4303	if ((pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
4304	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
4305	else
4306	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
4307	}
4308
4309
4310	/*
4311	*
4312	* Common opcode decoders.
4313	* Common opcode decoders.
4314	* Common opcode decoders.
4315	*
4316	*/
4317	//#include <iprt/mem.h>
4318
4319	/**
4320	* Used to add extra details about a stub case.
4321	* @param pIemCpu The IEM per CPU state.
4322	*/
4323	static void iemOpStubMsg2(PIEMCPU pIemCpu)
4324	{
4325	#if defined(LOG_ENABLED) && defined(IN_RING3)
4326	PVM pVM = IEMCPU_TO_VM(pIemCpu);
4327	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
4328	char szRegs[4096];
4329	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4330	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4331	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4332	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4333	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4334	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4335	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4336	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4337	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4338	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4339	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4340	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4341	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4342	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4343	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4344	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4345	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4346	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4347	" efer=%016VR{efer}\n"
4348	" pat=%016VR{pat}\n"
4349	" sf_mask=%016VR{sf_mask}\n"
4350	"krnl_gs_base=%016VR{krnl_gs_base}\n"
4351	" lstar=%016VR{lstar}\n"
4352	" star=%016VR{star} cstar=%016VR{cstar}\n"
4353	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4354	);
4355
4356	char szInstr[256];
4357	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4358	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
4359	szInstr, sizeof(szInstr), NULL);
4360
4361	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
4362	#else
4363	RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip);
4364	#endif
4365	}
4366
4367	/**
4368	* Complains about a stub.
4369	*
4370	* Providing two versions of this macro, one for daily use and one for use when
4371	* working on IEM.
4372	*/
4373	#if 0
4374	# define IEMOP_BITCH_ABOUT_STUB() \
4375	do { \
4376	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
4377	iemOpStubMsg2(pIemCpu); \
4378	RTAssertPanic(); \
4379	} while (0)
4380	#else
4381	# define IEMOP_BITCH_ABOUT_STUB() Log(("Stub: %s (line %d)\n", __FUNCTION__, __LINE__));
4382	#endif
4383
4384	/** Stubs an opcode. */
4385	#define FNIEMOP_STUB(a_Name) \
4386	FNIEMOP_DEF(a_Name) \
4387	{ \
4388	IEMOP_BITCH_ABOUT_STUB(); \
4389	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
4390	} \
4391	typedef int ignore_semicolon
4392
4393	/** Stubs an opcode. */
4394	#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
4395	FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
4396	{ \
4397	IEMOP_BITCH_ABOUT_STUB(); \
4398	NOREF(a_Name0); \
4399	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
4400	} \
4401	typedef int ignore_semicolon
4402
4403	/** Stubs an opcode which currently should raise \#UD. */
4404	#define FNIEMOP_UD_STUB(a_Name) \
4405	FNIEMOP_DEF(a_Name) \
4406	{ \
4407	Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
4408	return IEMOP_RAISE_INVALID_OPCODE(); \
4409	} \
4410	typedef int ignore_semicolon
4411
4412	/** Stubs an opcode which currently should raise \#UD. */
4413	#define FNIEMOP_UD_STUB_1(a_Name, a_Type0, a_Name0) \
4414	FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
4415	{ \
4416	NOREF(a_Name0); \
4417	Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
4418	return IEMOP_RAISE_INVALID_OPCODE(); \
4419	} \
4420	typedef int ignore_semicolon
4421
4422
4423
4424	/** @name Register Access.
4425	* @{
4426	*/
4427
4428	/**
4429	* Gets a reference (pointer) to the specified hidden segment register.
4430	*
4431	* @returns Hidden register reference.
4432	* @param pIemCpu The per CPU data.
4433	* @param iSegReg The segment register.
4434	*/
4435	static PCPUMSELREG iemSRegGetHid(PIEMCPU pIemCpu, uint8_t iSegReg)
4436	{
4437	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4438	PCPUMSELREG pSReg;
4439	switch (iSegReg)
4440	{
4441	case X86_SREG_ES: pSReg = &pCtx->es; break;
4442	case X86_SREG_CS: pSReg = &pCtx->cs; break;
4443	case X86_SREG_SS: pSReg = &pCtx->ss; break;
4444	case X86_SREG_DS: pSReg = &pCtx->ds; break;
4445	case X86_SREG_FS: pSReg = &pCtx->fs; break;
4446	case X86_SREG_GS: pSReg = &pCtx->gs; break;
4447	default:
4448	AssertFailedReturn(NULL);
4449	}
4450	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
4451	if (!CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), pSReg))
4452	CPUMGuestLazyLoadHiddenSelectorReg(IEMCPU_TO_VMCPU(pIemCpu), pSReg);
4453	#else
4454	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(IEMCPU_TO_VMCPU(pIemCpu), pSReg));
4455	#endif
4456	return pSReg;
4457	}
4458
4459
4460	/**
4461	* Gets a reference (pointer) to the specified segment register (the selector
4462	* value).
4463	*
4464	* @returns Pointer to the selector variable.
4465	* @param pIemCpu The per CPU data.
4466	* @param iSegReg The segment register.
4467	*/
4468	static uint16_t *iemSRegRef(PIEMCPU pIemCpu, uint8_t iSegReg)
4469	{
4470	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4471	switch (iSegReg)
4472	{
4473	case X86_SREG_ES: return &pCtx->es.Sel;
4474	case X86_SREG_CS: return &pCtx->cs.Sel;
4475	case X86_SREG_SS: return &pCtx->ss.Sel;
4476	case X86_SREG_DS: return &pCtx->ds.Sel;
4477	case X86_SREG_FS: return &pCtx->fs.Sel;
4478	case X86_SREG_GS: return &pCtx->gs.Sel;
4479	}
4480	AssertFailedReturn(NULL);
4481	}
4482
4483
4484	/**
4485	* Fetches the selector value of a segment register.
4486	*
4487	* @returns The selector value.
4488	* @param pIemCpu The per CPU data.
4489	* @param iSegReg The segment register.
4490	*/
4491	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg)
4492	{
4493	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4494	switch (iSegReg)
4495	{
4496	case X86_SREG_ES: return pCtx->es.Sel;
4497	case X86_SREG_CS: return pCtx->cs.Sel;
4498	case X86_SREG_SS: return pCtx->ss.Sel;
4499	case X86_SREG_DS: return pCtx->ds.Sel;
4500	case X86_SREG_FS: return pCtx->fs.Sel;
4501	case X86_SREG_GS: return pCtx->gs.Sel;
4502	}
4503	AssertFailedReturn(0xffff);
4504	}
4505
4506
4507	/**
4508	* Gets a reference (pointer) to the specified general register.
4509	*
4510	* @returns Register reference.
4511	* @param pIemCpu The per CPU data.
4512	* @param iReg The general register.
4513	*/
4514	static void *iemGRegRef(PIEMCPU pIemCpu, uint8_t iReg)
4515	{
4516	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4517	switch (iReg)
4518	{
4519	case X86_GREG_xAX: return &pCtx->rax;
4520	case X86_GREG_xCX: return &pCtx->rcx;
4521	case X86_GREG_xDX: return &pCtx->rdx;
4522	case X86_GREG_xBX: return &pCtx->rbx;
4523	case X86_GREG_xSP: return &pCtx->rsp;
4524	case X86_GREG_xBP: return &pCtx->rbp;
4525	case X86_GREG_xSI: return &pCtx->rsi;
4526	case X86_GREG_xDI: return &pCtx->rdi;
4527	case X86_GREG_x8: return &pCtx->r8;
4528	case X86_GREG_x9: return &pCtx->r9;
4529	case X86_GREG_x10: return &pCtx->r10;
4530	case X86_GREG_x11: return &pCtx->r11;
4531	case X86_GREG_x12: return &pCtx->r12;
4532	case X86_GREG_x13: return &pCtx->r13;
4533	case X86_GREG_x14: return &pCtx->r14;
4534	case X86_GREG_x15: return &pCtx->r15;
4535	}
4536	AssertFailedReturn(NULL);
4537	}
4538
4539
4540	/**
4541	* Gets a reference (pointer) to the specified 8-bit general register.
4542	*
4543	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
4544	*
4545	* @returns Register reference.
4546	* @param pIemCpu The per CPU data.
4547	* @param iReg The register.
4548	*/
4549	static uint8_t *iemGRegRefU8(PIEMCPU pIemCpu, uint8_t iReg)
4550	{
4551	if (pIemCpu->fPrefixes & IEM_OP_PRF_REX)
4552	return (uint8_t *)iemGRegRef(pIemCpu, iReg);
4553
4554	uint8_t pu8Reg = (uint8_t )iemGRegRef(pIemCpu, iReg & 3);
4555	if (iReg >= 4)
4556	pu8Reg++;
4557	return pu8Reg;
4558	}
4559
4560
4561	/**
4562	* Fetches the value of a 8-bit general register.
4563	*
4564	* @returns The register value.
4565	* @param pIemCpu The per CPU data.
4566	* @param iReg The register.
4567	*/
4568	static uint8_t iemGRegFetchU8(PIEMCPU pIemCpu, uint8_t iReg)
4569	{
4570	uint8_t const *pbSrc = iemGRegRefU8(pIemCpu, iReg);
4571	return *pbSrc;
4572	}
4573
4574
4575	/**
4576	* Fetches the value of a 16-bit general register.
4577	*
4578	* @returns The register value.
4579	* @param pIemCpu The per CPU data.
4580	* @param iReg The register.
4581	*/
4582	static uint16_t iemGRegFetchU16(PIEMCPU pIemCpu, uint8_t iReg)
4583	{
4584	return (uint16_t )iemGRegRef(pIemCpu, iReg);
4585	}
4586
4587
4588	/**
4589	* Fetches the value of a 32-bit general register.
4590	*
4591	* @returns The register value.
4592	* @param pIemCpu The per CPU data.
4593	* @param iReg The register.
4594	*/
4595	static uint32_t iemGRegFetchU32(PIEMCPU pIemCpu, uint8_t iReg)
4596	{
4597	return (uint32_t )iemGRegRef(pIemCpu, iReg);
4598	}
4599
4600
4601	/**
4602	* Fetches the value of a 64-bit general register.
4603	*
4604	* @returns The register value.
4605	* @param pIemCpu The per CPU data.
4606	* @param iReg The register.
4607	*/
4608	static uint64_t iemGRegFetchU64(PIEMCPU pIemCpu, uint8_t iReg)
4609	{
4610	return (uint64_t )iemGRegRef(pIemCpu, iReg);
4611	}
4612
4613
4614	/**
4615	* Is the FPU state in FXSAVE format or not.
4616	*
4617	* @returns true if it is, false if it's in FNSAVE.
4618	* @param pVCpu Pointer to the VMCPU.
4619	*/
4620	DECLINLINE(bool) iemFRegIsFxSaveFormat(PIEMCPU pIemCpu)
4621	{
4622	#ifdef RT_ARCH_AMD64
4623	NOREF(pIemCpu);
4624	return true;
4625	#else
4626	NOREF(pIemCpu); /// @todo return pVCpu->pVMR3->cpum.s.CPUFeatures.edx.u1FXSR;
4627	return true;
4628	#endif
4629	}
4630
4631
4632	/**
4633	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
4634	*
4635	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4636	* segment limit.
4637	*
4638	* @param pIemCpu The per CPU data.
4639	* @param offNextInstr The offset of the next instruction.
4640	*/
4641	static VBOXSTRICTRC iemRegRipRelativeJumpS8(PIEMCPU pIemCpu, int8_t offNextInstr)
4642	{
4643	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4644	switch (pIemCpu->enmEffOpSize)
4645	{
4646	case IEMMODE_16BIT:
4647	{
4648	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
4649	if ( uNewIp > pCtx->cs.u32Limit
4650	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4651	return iemRaiseGeneralProtectionFault0(pIemCpu);
4652	pCtx->rip = uNewIp;
4653	break;
4654	}
4655
4656	case IEMMODE_32BIT:
4657	{
4658	Assert(pCtx->rip <= UINT32_MAX);
4659	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
4660
4661	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
4662	if (uNewEip > pCtx->cs.u32Limit)
4663	return iemRaiseGeneralProtectionFault0(pIemCpu);
4664	pCtx->rip = uNewEip;
4665	break;
4666	}
4667
4668	case IEMMODE_64BIT:
4669	{
4670	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
4671
4672	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
4673	if (!IEM_IS_CANONICAL(uNewRip))
4674	return iemRaiseGeneralProtectionFault0(pIemCpu);
4675	pCtx->rip = uNewRip;
4676	break;
4677	}
4678
4679	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4680	}
4681
4682	pCtx->eflags.Bits.u1RF = 0;
4683	return VINF_SUCCESS;
4684	}
4685
4686
4687	/**
4688	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
4689	*
4690	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4691	* segment limit.
4692	*
4693	* @returns Strict VBox status code.
4694	* @param pIemCpu The per CPU data.
4695	* @param offNextInstr The offset of the next instruction.
4696	*/
4697	static VBOXSTRICTRC iemRegRipRelativeJumpS16(PIEMCPU pIemCpu, int16_t offNextInstr)
4698	{
4699	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4700	Assert(pIemCpu->enmEffOpSize == IEMMODE_16BIT);
4701
4702	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
4703	if ( uNewIp > pCtx->cs.u32Limit
4704	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4705	return iemRaiseGeneralProtectionFault0(pIemCpu);
4706	/** @todo Test 16-bit jump in 64-bit mode. possible? */
4707	pCtx->rip = uNewIp;
4708	pCtx->eflags.Bits.u1RF = 0;
4709
4710	return VINF_SUCCESS;
4711	}
4712
4713
4714	/**
4715	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
4716	*
4717	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4718	* segment limit.
4719	*
4720	* @returns Strict VBox status code.
4721	* @param pIemCpu The per CPU data.
4722	* @param offNextInstr The offset of the next instruction.
4723	*/
4724	static VBOXSTRICTRC iemRegRipRelativeJumpS32(PIEMCPU pIemCpu, int32_t offNextInstr)
4725	{
4726	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4727	Assert(pIemCpu->enmEffOpSize != IEMMODE_16BIT);
4728
4729	if (pIemCpu->enmEffOpSize == IEMMODE_32BIT)
4730	{
4731	Assert(pCtx->rip <= UINT32_MAX); Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
4732
4733	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
4734	if (uNewEip > pCtx->cs.u32Limit)
4735	return iemRaiseGeneralProtectionFault0(pIemCpu);
4736	pCtx->rip = uNewEip;
4737	}
4738	else
4739	{
4740	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
4741
4742	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
4743	if (!IEM_IS_CANONICAL(uNewRip))
4744	return iemRaiseGeneralProtectionFault0(pIemCpu);
4745	pCtx->rip = uNewRip;
4746	}
4747	pCtx->eflags.Bits.u1RF = 0;
4748	return VINF_SUCCESS;
4749	}
4750
4751
4752	/**
4753	* Performs a near jump to the specified address.
4754	*
4755	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4756	* segment limit.
4757	*
4758	* @param pIemCpu The per CPU data.
4759	* @param uNewRip The new RIP value.
4760	*/
4761	static VBOXSTRICTRC iemRegRipJump(PIEMCPU pIemCpu, uint64_t uNewRip)
4762	{
4763	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4764	switch (pIemCpu->enmEffOpSize)
4765	{
4766	case IEMMODE_16BIT:
4767	{
4768	Assert(uNewRip <= UINT16_MAX);
4769	if ( uNewRip > pCtx->cs.u32Limit
4770	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4771	return iemRaiseGeneralProtectionFault0(pIemCpu);
4772	/** @todo Test 16-bit jump in 64-bit mode. */
4773	pCtx->rip = uNewRip;
4774	break;
4775	}
4776
4777	case IEMMODE_32BIT:
4778	{
4779	Assert(uNewRip <= UINT32_MAX);
4780	Assert(pCtx->rip <= UINT32_MAX);
4781	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
4782
4783	if (uNewRip > pCtx->cs.u32Limit)
4784	return iemRaiseGeneralProtectionFault0(pIemCpu);
4785	pCtx->rip = uNewRip;
4786	break;
4787	}
4788
4789	case IEMMODE_64BIT:
4790	{
4791	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
4792
4793	if (!IEM_IS_CANONICAL(uNewRip))
4794	return iemRaiseGeneralProtectionFault0(pIemCpu);
4795	pCtx->rip = uNewRip;
4796	break;
4797	}
4798
4799	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4800	}
4801
4802	pCtx->eflags.Bits.u1RF = 0;
4803	return VINF_SUCCESS;
4804	}
4805
4806
4807	/**
4808	* Get the address of the top of the stack.
4809	*
4810	* @param pIemCpu The per CPU data.
4811	* @param pCtx The CPU context which SP/ESP/RSP should be
4812	* read.
4813	*/
4814	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCIEMCPU pIemCpu, PCCPUMCTX pCtx)
4815	{
4816	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
4817	return pCtx->rsp;
4818	if (pCtx->ss.Attr.n.u1DefBig)
4819	return pCtx->esp;
4820	return pCtx->sp;
4821	}
4822
4823
4824	/**
4825	* Updates the RIP/EIP/IP to point to the next instruction.
4826	*
4827	* This function leaves the EFLAGS.RF flag alone.
4828	*
4829	* @param pIemCpu The per CPU data.
4830	* @param cbInstr The number of bytes to add.
4831	*/
4832	static void iemRegAddToRipKeepRF(PIEMCPU pIemCpu, uint8_t cbInstr)
4833	{
4834	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4835	switch (pIemCpu->enmCpuMode)
4836	{
4837	case IEMMODE_16BIT:
4838	Assert(pCtx->rip <= UINT16_MAX);
4839	pCtx->eip += cbInstr;
4840	pCtx->eip &= UINT32_C(0xffff);
4841	break;
4842
4843	case IEMMODE_32BIT:
4844	pCtx->eip += cbInstr;
4845	Assert(pCtx->rip <= UINT32_MAX);
4846	break;
4847
4848	case IEMMODE_64BIT:
4849	pCtx->rip += cbInstr;
4850	break;
4851	default: AssertFailed();
4852	}
4853	}
4854
4855
4856	#if 0
4857	/**
4858	* Updates the RIP/EIP/IP to point to the next instruction.
4859	*
4860	* @param pIemCpu The per CPU data.
4861	*/
4862	static void iemRegUpdateRipKeepRF(PIEMCPU pIemCpu)
4863	{
4864	return iemRegAddToRipKeepRF(pIemCpu, pIemCpu->offOpcode);
4865	}
4866	#endif
4867
4868
4869
4870	/**
4871	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
4872	*
4873	* @param pIemCpu The per CPU data.
4874	* @param cbInstr The number of bytes to add.
4875	*/
4876	static void iemRegAddToRipAndClearRF(PIEMCPU pIemCpu, uint8_t cbInstr)
4877	{
4878	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4879
4880	pCtx->eflags.Bits.u1RF = 0;
4881
4882	switch (pIemCpu->enmCpuMode)
4883	{
4884	case IEMMODE_16BIT:
4885	Assert(pCtx->rip <= UINT16_MAX);
4886	pCtx->eip += cbInstr;
4887	pCtx->eip &= UINT32_C(0xffff);
4888	break;
4889
4890	case IEMMODE_32BIT:
4891	pCtx->eip += cbInstr;
4892	Assert(pCtx->rip <= UINT32_MAX);
4893	break;
4894
4895	case IEMMODE_64BIT:
4896	pCtx->rip += cbInstr;
4897	break;
4898	default: AssertFailed();
4899	}
4900	}
4901
4902
4903	/**
4904	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
4905	*
4906	* @param pIemCpu The per CPU data.
4907	*/
4908	static void iemRegUpdateRipAndClearRF(PIEMCPU pIemCpu)
4909	{
4910	return iemRegAddToRipAndClearRF(pIemCpu, pIemCpu->offOpcode);
4911	}
4912
4913
4914	/**
4915	* Adds to the stack pointer.
4916	*
4917	* @param pIemCpu The per CPU data.
4918	* @param pCtx The CPU context which SP/ESP/RSP should be
4919	* updated.
4920	* @param cbToAdd The number of bytes to add.
4921	*/
4922	DECLINLINE(void) iemRegAddToRsp(PCIEMCPU pIemCpu, PCPUMCTX pCtx, uint8_t cbToAdd)
4923	{
4924	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
4925	pCtx->rsp += cbToAdd;
4926	else if (pCtx->ss.Attr.n.u1DefBig)
4927	pCtx->esp += cbToAdd;
4928	else
4929	pCtx->sp += cbToAdd;
4930	}
4931
4932
4933	/**
4934	* Subtracts from the stack pointer.
4935	*
4936	* @param pIemCpu The per CPU data.
4937	* @param pCtx The CPU context which SP/ESP/RSP should be
4938	* updated.
4939	* @param cbToSub The number of bytes to subtract.
4940	*/
4941	DECLINLINE(void) iemRegSubFromRsp(PCIEMCPU pIemCpu, PCPUMCTX pCtx, uint8_t cbToSub)
4942	{
4943	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
4944	pCtx->rsp -= cbToSub;
4945	else if (pCtx->ss.Attr.n.u1DefBig)
4946	pCtx->esp -= cbToSub;
4947	else
4948	pCtx->sp -= cbToSub;
4949	}
4950
4951
4952	/**
4953	* Adds to the temporary stack pointer.
4954	*
4955	* @param pIemCpu The per CPU data.
4956	* @param pTmpRsp The temporary SP/ESP/RSP to update.
4957	* @param cbToAdd The number of bytes to add.
4958	* @param pCtx Where to get the current stack mode.
4959	*/
4960	DECLINLINE(void) iemRegAddToRspEx(PCIEMCPU pIemCpu, PCCPUMCTX pCtx, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
4961	{
4962	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
4963	pTmpRsp->u += cbToAdd;
4964	else if (pCtx->ss.Attr.n.u1DefBig)
4965	pTmpRsp->DWords.dw0 += cbToAdd;
4966	else
4967	pTmpRsp->Words.w0 += cbToAdd;
4968	}
4969
4970
4971	/**
4972	* Subtracts from the temporary stack pointer.
4973	*
4974	* @param pIemCpu The per CPU data.
4975	* @param pTmpRsp The temporary SP/ESP/RSP to update.
4976	* @param cbToSub The number of bytes to subtract.
4977	* @param pCtx Where to get the current stack mode.
4978	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
4979	* expecting that.
4980	*/
4981	DECLINLINE(void) iemRegSubFromRspEx(PCIEMCPU pIemCpu, PCCPUMCTX pCtx, PRTUINT64U pTmpRsp, uint16_t cbToSub)
4982	{
4983	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
4984	pTmpRsp->u -= cbToSub;
4985	else if (pCtx->ss.Attr.n.u1DefBig)
4986	pTmpRsp->DWords.dw0 -= cbToSub;
4987	else
4988	pTmpRsp->Words.w0 -= cbToSub;
4989	}
4990
4991
4992	/**
4993	* Calculates the effective stack address for a push of the specified size as
4994	* well as the new RSP value (upper bits may be masked).
4995	*
4996	* @returns Effective stack addressf for the push.
4997	* @param pIemCpu The IEM per CPU data.
4998	* @param pCtx Where to get the current stack mode.
4999	* @param cbItem The size of the stack item to pop.
5000	* @param puNewRsp Where to return the new RSP value.
5001	*/
5002	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
5003	{
5004	RTUINT64U uTmpRsp;
5005	RTGCPTR GCPtrTop;
5006	uTmpRsp.u = pCtx->rsp;
5007
5008	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
5009	GCPtrTop = uTmpRsp.u -= cbItem;
5010	else if (pCtx->ss.Attr.n.u1DefBig)
5011	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
5012	else
5013	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
5014	*puNewRsp = uTmpRsp.u;
5015	return GCPtrTop;
5016	}
5017
5018
5019	/**
5020	* Gets the current stack pointer and calculates the value after a pop of the
5021	* specified size.
5022	*
5023	* @returns Current stack pointer.
5024	* @param pIemCpu The per CPU data.
5025	* @param pCtx Where to get the current stack mode.
5026	* @param cbItem The size of the stack item to pop.
5027	* @param puNewRsp Where to return the new RSP value.
5028	*/
5029	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
5030	{
5031	RTUINT64U uTmpRsp;
5032	RTGCPTR GCPtrTop;
5033	uTmpRsp.u = pCtx->rsp;
5034
5035	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
5036	{
5037	GCPtrTop = uTmpRsp.u;
5038	uTmpRsp.u += cbItem;
5039	}
5040	else if (pCtx->ss.Attr.n.u1DefBig)
5041	{
5042	GCPtrTop = uTmpRsp.DWords.dw0;
5043	uTmpRsp.DWords.dw0 += cbItem;
5044	}
5045	else
5046	{
5047	GCPtrTop = uTmpRsp.Words.w0;
5048	uTmpRsp.Words.w0 += cbItem;
5049	}
5050	*puNewRsp = uTmpRsp.u;
5051	return GCPtrTop;
5052	}
5053
5054
5055	/**
5056	* Calculates the effective stack address for a push of the specified size as
5057	* well as the new temporary RSP value (upper bits may be masked).
5058	*
5059	* @returns Effective stack addressf for the push.
5060	* @param pIemCpu The per CPU data.
5061	* @param pTmpRsp The temporary stack pointer. This is updated.
5062	* @param cbItem The size of the stack item to pop.
5063	* @param puNewRsp Where to return the new RSP value.
5064	*/
5065	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCIEMCPU pIemCpu, PCCPUMCTX pCtx, PRTUINT64U pTmpRsp, uint8_t cbItem)
5066	{
5067	RTGCPTR GCPtrTop;
5068
5069	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
5070	GCPtrTop = pTmpRsp->u -= cbItem;
5071	else if (pCtx->ss.Attr.n.u1DefBig)
5072	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
5073	else
5074	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
5075	return GCPtrTop;
5076	}
5077
5078
5079	/**
5080	* Gets the effective stack address for a pop of the specified size and
5081	* calculates and updates the temporary RSP.
5082	*
5083	* @returns Current stack pointer.
5084	* @param pIemCpu The per CPU data.
5085	* @param pTmpRsp The temporary stack pointer. This is updated.
5086	* @param pCtx Where to get the current stack mode.
5087	* @param cbItem The size of the stack item to pop.
5088	*/
5089	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCIEMCPU pIemCpu, PCCPUMCTX pCtx, PRTUINT64U pTmpRsp, uint8_t cbItem)
5090	{
5091	RTGCPTR GCPtrTop;
5092	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
5093	{
5094	GCPtrTop = pTmpRsp->u;
5095	pTmpRsp->u += cbItem;
5096	}
5097	else if (pCtx->ss.Attr.n.u1DefBig)
5098	{
5099	GCPtrTop = pTmpRsp->DWords.dw0;
5100	pTmpRsp->DWords.dw0 += cbItem;
5101	}
5102	else
5103	{
5104	GCPtrTop = pTmpRsp->Words.w0;
5105	pTmpRsp->Words.w0 += cbItem;
5106	}
5107	return GCPtrTop;
5108	}
5109
5110
5111	/**
5112	* Checks if an Intel CPUID feature bit is set.
5113	*
5114	* @returns true / false.
5115	*
5116	* @param pIemCpu The IEM per CPU data.
5117	* @param fEdx The EDX bit to test, or 0 if ECX.
5118	* @param fEcx The ECX bit to test, or 0 if EDX.
5119	* @remarks Used via IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX,
5120	* IEM_IS_INTEL_CPUID_FEATURE_PRESENT_ECX and others.
5121	*/
5122	static bool iemRegIsIntelCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
5123	{
5124	uint32_t uEax, uEbx, uEcx, uEdx;
5125	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x00000001, &uEax, &uEbx, &uEcx, &uEdx);
5126	return (fEcx && (uEcx & fEcx))
5127	\|\| (fEdx && (uEdx & fEdx));
5128	}
5129
5130
5131	/**
5132	* Checks if an AMD CPUID feature bit is set.
5133	*
5134	* @returns true / false.
5135	*
5136	* @param pIemCpu The IEM per CPU data.
5137	* @param fEdx The EDX bit to test, or 0 if ECX.
5138	* @param fEcx The ECX bit to test, or 0 if EDX.
5139	* @remarks Used via IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX,
5140	* IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX and others.
5141	*/
5142	static bool iemRegIsAmdCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
5143	{
5144	uint32_t uEax, uEbx, uEcx, uEdx;
5145	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x80000001, &uEax, &uEbx, &uEcx, &uEdx);
5146	return (fEcx && (uEcx & fEcx))
5147	\|\| (fEdx && (uEdx & fEdx));
5148	}
5149
5150	/** @} */
5151
5152
5153	/** @name FPU access and helpers.
5154	*
5155	* @{
5156	*/
5157
5158
5159	/**
5160	* Hook for preparing to use the host FPU.
5161	*
5162	* This is necessary in ring-0 and raw-mode context.
5163	*
5164	* @param pIemCpu The IEM per CPU data.
5165	*/
5166	DECLINLINE(void) iemFpuPrepareUsage(PIEMCPU pIemCpu)
5167	{
5168	#ifdef IN_RING3
5169	NOREF(pIemCpu);
5170	#else
5171	/** @todo RZ: FIXME */
5172	//# error "Implement me"
5173	#endif
5174	}
5175
5176
5177	/**
5178	* Hook for preparing to use the host FPU for SSE
5179	*
5180	* This is necessary in ring-0 and raw-mode context.
5181	*
5182	* @param pIemCpu The IEM per CPU data.
5183	*/
5184	DECLINLINE(void) iemFpuPrepareUsageSse(PIEMCPU pIemCpu)
5185	{
5186	iemFpuPrepareUsage(pIemCpu);
5187	}
5188
5189
5190	/**
5191	* Stores a QNaN value into a FPU register.
5192	*
5193	* @param pReg Pointer to the register.
5194	*/
5195	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
5196	{
5197	pReg->au32[0] = UINT32_C(0x00000000);
5198	pReg->au32[1] = UINT32_C(0xc0000000);
5199	pReg->au16[4] = UINT16_C(0xffff);
5200	}
5201
5202
5203	/**
5204	* Updates the FOP, FPU.CS and FPUIP registers.
5205	*
5206	* @param pIemCpu The IEM per CPU data.
5207	* @param pCtx The CPU context.
5208	*/
5209	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PIEMCPU pIemCpu, PCPUMCTX pCtx)
5210	{
5211	pCtx->fpu.FOP = pIemCpu->abOpcode[pIemCpu->offFpuOpcode]
5212	\| ((uint16_t)(pIemCpu->abOpcode[pIemCpu->offFpuOpcode - 1] & 0x7) << 8);
5213	/** @todo FPU.CS and FPUIP needs to be kept seperately. */
5214	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu))
5215	{
5216	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
5217	* happens in real mode here based on the fnsave and fnstenv images. */
5218	pCtx->fpu.CS = 0;
5219	pCtx->fpu.FPUIP = pCtx->eip \| ((uint32_t)pCtx->cs.Sel << 4);
5220	}
5221	else
5222	{
5223	pCtx->fpu.CS = pCtx->cs.Sel;
5224	pCtx->fpu.FPUIP = pCtx->rip;
5225	}
5226	}
5227
5228
5229	/**
5230	* Updates the FPU.DS and FPUDP registers.
5231	*
5232	* @param pIemCpu The IEM per CPU data.
5233	* @param pCtx The CPU context.
5234	* @param iEffSeg The effective segment register.
5235	* @param GCPtrEff The effective address relative to @a iEffSeg.
5236	*/
5237	DECLINLINE(void) iemFpuUpdateDP(PIEMCPU pIemCpu, PCPUMCTX pCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5238	{
5239	RTSEL sel;
5240	switch (iEffSeg)
5241	{
5242	case X86_SREG_DS: sel = pCtx->ds.Sel; break;
5243	case X86_SREG_SS: sel = pCtx->ss.Sel; break;
5244	case X86_SREG_CS: sel = pCtx->cs.Sel; break;
5245	case X86_SREG_ES: sel = pCtx->es.Sel; break;
5246	case X86_SREG_FS: sel = pCtx->fs.Sel; break;
5247	case X86_SREG_GS: sel = pCtx->gs.Sel; break;
5248	default:
5249	AssertMsgFailed(("%d\n", iEffSeg));
5250	sel = pCtx->ds.Sel;
5251	}
5252	/** @todo FPU.DS and FPUDP needs to be kept seperately. */
5253	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu))
5254	{
5255	pCtx->fpu.DS = 0;
5256	pCtx->fpu.FPUDP = (uint32_t)GCPtrEff \| ((uint32_t)sel << 4);
5257	}
5258	else
5259	{
5260	pCtx->fpu.DS = sel;
5261	pCtx->fpu.FPUDP = GCPtrEff;
5262	}
5263	}
5264
5265
5266	/**
5267	* Rotates the stack registers in the push direction.
5268	*
5269	* @param pCtx The CPU context.
5270	* @remarks This is a complete waste of time, but fxsave stores the registers in
5271	* stack order.
5272	*/
5273	DECLINLINE(void) iemFpuRotateStackPush(PCPUMCTX pCtx)
5274	{
5275	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[7].r80;
5276	pCtx->fpu.aRegs[7].r80 = pCtx->fpu.aRegs[6].r80;
5277	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[5].r80;
5278	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[4].r80;
5279	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[3].r80;
5280	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[2].r80;
5281	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[1].r80;
5282	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[0].r80;
5283	pCtx->fpu.aRegs[0].r80 = r80Tmp;
5284	}
5285
5286
5287	/**
5288	* Rotates the stack registers in the pop direction.
5289	*
5290	* @param pCtx The CPU context.
5291	* @remarks This is a complete waste of time, but fxsave stores the registers in
5292	* stack order.
5293	*/
5294	DECLINLINE(void) iemFpuRotateStackPop(PCPUMCTX pCtx)
5295	{
5296	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[0].r80;
5297	pCtx->fpu.aRegs[0].r80 = pCtx->fpu.aRegs[1].r80;
5298	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[2].r80;
5299	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[3].r80;
5300	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[4].r80;
5301	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[5].r80;
5302	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[6].r80;
5303	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[7].r80;
5304	pCtx->fpu.aRegs[7].r80 = r80Tmp;
5305	}
5306
5307
5308	/**
5309	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
5310	* exception prevents it.
5311	*
5312	* @param pIemCpu The IEM per CPU data.
5313	* @param pResult The FPU operation result to push.
5314	* @param pCtx The CPU context.
5315	*/
5316	static void iemFpuMaybePushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, PCPUMCTX pCtx)
5317	{
5318	/* Update FSW and bail if there are pending exceptions afterwards. */
5319	uint16_t fFsw = pCtx->fpu.FSW & ~X86_FSW_C_MASK;
5320	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
5321	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
5322	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
5323	{
5324	pCtx->fpu.FSW = fFsw;
5325	return;
5326	}
5327
5328	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
5329	if (!(pCtx->fpu.FTW & RT_BIT(iNewTop)))
5330	{
5331	/* All is fine, push the actual value. */
5332	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
5333	pCtx->fpu.aRegs[7].r80 = pResult->r80Result;
5334	}
5335	else if (pCtx->fpu.FCW & X86_FCW_IM)
5336	{
5337	/* Masked stack overflow, push QNaN. */
5338	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
5339	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
5340	}
5341	else
5342	{
5343	/* Raise stack overflow, don't push anything. */
5344	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
5345	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
5346	return;
5347	}
5348
5349	fFsw &= ~X86_FSW_TOP_MASK;
5350	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
5351	pCtx->fpu.FSW = fFsw;
5352
5353	iemFpuRotateStackPush(pCtx);
5354	}
5355
5356
5357	/**
5358	* Stores a result in a FPU register and updates the FSW and FTW.
5359	*
5360	* @param pIemCpu The IEM per CPU data.
5361	* @param pResult The result to store.
5362	* @param iStReg Which FPU register to store it in.
5363	* @param pCtx The CPU context.
5364	*/
5365	static void iemFpuStoreResultOnly(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, PCPUMCTX pCtx)
5366	{
5367	Assert(iStReg < 8);
5368	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
5369	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5370	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_TOP_MASK;
5371	pCtx->fpu.FTW \|= RT_BIT(iReg);
5372	pCtx->fpu.aRegs[iStReg].r80 = pResult->r80Result;
5373	}
5374
5375
5376	/**
5377	* Only updates the FPU status word (FSW) with the result of the current
5378	* instruction.
5379	*
5380	* @param pCtx The CPU context.
5381	* @param u16FSW The FSW output of the current instruction.
5382	*/
5383	static void iemFpuUpdateFSWOnly(PCPUMCTX pCtx, uint16_t u16FSW)
5384	{
5385	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5386	pCtx->fpu.FSW \|= u16FSW & ~X86_FSW_TOP_MASK;
5387	}
5388
5389
5390	/**
5391	* Pops one item off the FPU stack if no pending exception prevents it.
5392	*
5393	* @param pCtx The CPU context.
5394	*/
5395	static void iemFpuMaybePopOne(PCPUMCTX pCtx)
5396	{
5397	/* Check pending exceptions. */
5398	uint16_t uFSW = pCtx->fpu.FSW;
5399	if ( (pCtx->fpu.FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
5400	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
5401	return;
5402
5403	/* TOP--. */
5404	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
5405	uFSW &= ~X86_FSW_TOP_MASK;
5406	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
5407	pCtx->fpu.FSW = uFSW;
5408
5409	/* Mark the previous ST0 as empty. */
5410	iOldTop >>= X86_FSW_TOP_SHIFT;
5411	pCtx->fpu.FTW &= ~RT_BIT(iOldTop);
5412
5413	/* Rotate the registers. */
5414	iemFpuRotateStackPop(pCtx);
5415	}
5416
5417
5418	/**
5419	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
5420	*
5421	* @param pIemCpu The IEM per CPU data.
5422	* @param pResult The FPU operation result to push.
5423	*/
5424	static void iemFpuPushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult)
5425	{
5426	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5427	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5428	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
5429	}
5430
5431
5432	/**
5433	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
5434	* and sets FPUDP and FPUDS.
5435	*
5436	* @param pIemCpu The IEM per CPU data.
5437	* @param pResult The FPU operation result to push.
5438	* @param iEffSeg The effective segment register.
5439	* @param GCPtrEff The effective address relative to @a iEffSeg.
5440	*/
5441	static void iemFpuPushResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5442	{
5443	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5444	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5445	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5446	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
5447	}
5448
5449
5450	/**
5451	* Replace ST0 with the first value and push the second onto the FPU stack,
5452	* unless a pending exception prevents it.
5453	*
5454	* @param pIemCpu The IEM per CPU data.
5455	* @param pResult The FPU operation result to store and push.
5456	*/
5457	static void iemFpuPushResultTwo(PIEMCPU pIemCpu, PIEMFPURESULTTWO pResult)
5458	{
5459	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5460	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5461
5462	/* Update FSW and bail if there are pending exceptions afterwards. */
5463	uint16_t fFsw = pCtx->fpu.FSW & ~X86_FSW_C_MASK;
5464	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
5465	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
5466	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
5467	{
5468	pCtx->fpu.FSW = fFsw;
5469	return;
5470	}
5471
5472	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
5473	if (!(pCtx->fpu.FTW & RT_BIT(iNewTop)))
5474	{
5475	/* All is fine, push the actual value. */
5476	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
5477	pCtx->fpu.aRegs[0].r80 = pResult->r80Result1;
5478	pCtx->fpu.aRegs[7].r80 = pResult->r80Result2;
5479	}
5480	else if (pCtx->fpu.FCW & X86_FCW_IM)
5481	{
5482	/* Masked stack overflow, push QNaN. */
5483	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
5484	iemFpuStoreQNan(&pCtx->fpu.aRegs[0].r80);
5485	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
5486	}
5487	else
5488	{
5489	/* Raise stack overflow, don't push anything. */
5490	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
5491	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
5492	return;
5493	}
5494
5495	fFsw &= ~X86_FSW_TOP_MASK;
5496	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
5497	pCtx->fpu.FSW = fFsw;
5498
5499	iemFpuRotateStackPush(pCtx);
5500	}
5501
5502
5503	/**
5504	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
5505	* FOP.
5506	*
5507	* @param pIemCpu The IEM per CPU data.
5508	* @param pResult The result to store.
5509	* @param iStReg Which FPU register to store it in.
5510	* @param pCtx The CPU context.
5511	*/
5512	static void iemFpuStoreResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
5513	{
5514	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5515	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5516	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
5517	}
5518
5519
5520	/**
5521	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
5522	* FOP, and then pops the stack.
5523	*
5524	* @param pIemCpu The IEM per CPU data.
5525	* @param pResult The result to store.
5526	* @param iStReg Which FPU register to store it in.
5527	* @param pCtx The CPU context.
5528	*/
5529	static void iemFpuStoreResultThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
5530	{
5531	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5532	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5533	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
5534	iemFpuMaybePopOne(pCtx);
5535	}
5536
5537
5538	/**
5539	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
5540	* FPUDP, and FPUDS.
5541	*
5542	* @param pIemCpu The IEM per CPU data.
5543	* @param pResult The result to store.
5544	* @param iStReg Which FPU register to store it in.
5545	* @param pCtx The CPU context.
5546	* @param iEffSeg The effective memory operand selector register.
5547	* @param GCPtrEff The effective memory operand offset.
5548	*/
5549	static void iemFpuStoreResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5550	{
5551	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5552	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
5553	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5554	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
5555	}
5556
5557
5558	/**
5559	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
5560	* FPUDP, and FPUDS, and then pops the stack.
5561	*
5562	* @param pIemCpu The IEM per CPU data.
5563	* @param pResult The result to store.
5564	* @param iStReg Which FPU register to store it in.
5565	* @param pCtx The CPU context.
5566	* @param iEffSeg The effective memory operand selector register.
5567	* @param GCPtrEff The effective memory operand offset.
5568	*/
5569	static void iemFpuStoreResultWithMemOpThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult,
5570	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5571	{
5572	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5573	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5574	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5575	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
5576	iemFpuMaybePopOne(pCtx);
5577	}
5578
5579
5580	/**
5581	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
5582	*
5583	* @param pIemCpu The IEM per CPU data.
5584	*/
5585	static void iemFpuUpdateOpcodeAndIp(PIEMCPU pIemCpu)
5586	{
5587	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pIemCpu->CTX_SUFF(pCtx));
5588	}
5589
5590
5591	/**
5592	* Marks the specified stack register as free (for FFREE).
5593	*
5594	* @param pIemCpu The IEM per CPU data.
5595	* @param iStReg The register to free.
5596	*/
5597	static void iemFpuStackFree(PIEMCPU pIemCpu, uint8_t iStReg)
5598	{
5599	Assert(iStReg < 8);
5600	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5601	uint8_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
5602	pCtx->fpu.FTW &= ~RT_BIT(iReg);
5603	}
5604
5605
5606	/**
5607	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
5608	*
5609	* @param pIemCpu The IEM per CPU data.
5610	*/
5611	static void iemFpuStackIncTop(PIEMCPU pIemCpu)
5612	{
5613	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5614	uint16_t uFsw = pCtx->fpu.FSW;
5615	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
5616	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
5617	uFsw &= ~X86_FSW_TOP_MASK;
5618	uFsw \|= uTop;
5619	pCtx->fpu.FSW = uFsw;
5620	}
5621
5622
5623	/**
5624	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
5625	*
5626	* @param pIemCpu The IEM per CPU data.
5627	*/
5628	static void iemFpuStackDecTop(PIEMCPU pIemCpu)
5629	{
5630	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5631	uint16_t uFsw = pCtx->fpu.FSW;
5632	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
5633	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
5634	uFsw &= ~X86_FSW_TOP_MASK;
5635	uFsw \|= uTop;
5636	pCtx->fpu.FSW = uFsw;
5637	}
5638
5639
5640	/**
5641	* Updates the FSW, FOP, FPUIP, and FPUCS.
5642	*
5643	* @param pIemCpu The IEM per CPU data.
5644	* @param u16FSW The FSW from the current instruction.
5645	*/
5646	static void iemFpuUpdateFSW(PIEMCPU pIemCpu, uint16_t u16FSW)
5647	{
5648	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5649	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5650	iemFpuUpdateFSWOnly(pCtx, u16FSW);
5651	}
5652
5653
5654	/**
5655	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
5656	*
5657	* @param pIemCpu The IEM per CPU data.
5658	* @param u16FSW The FSW from the current instruction.
5659	*/
5660	static void iemFpuUpdateFSWThenPop(PIEMCPU pIemCpu, uint16_t u16FSW)
5661	{
5662	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5663	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5664	iemFpuUpdateFSWOnly(pCtx, u16FSW);
5665	iemFpuMaybePopOne(pCtx);
5666	}
5667
5668
5669	/**
5670	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
5671	*
5672	* @param pIemCpu The IEM per CPU data.
5673	* @param u16FSW The FSW from the current instruction.
5674	* @param iEffSeg The effective memory operand selector register.
5675	* @param GCPtrEff The effective memory operand offset.
5676	*/
5677	static void iemFpuUpdateFSWWithMemOp(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5678	{
5679	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5680	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5681	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5682	iemFpuUpdateFSWOnly(pCtx, u16FSW);
5683	}
5684
5685
5686	/**
5687	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
5688	*
5689	* @param pIemCpu The IEM per CPU data.
5690	* @param u16FSW The FSW from the current instruction.
5691	*/
5692	static void iemFpuUpdateFSWThenPopPop(PIEMCPU pIemCpu, uint16_t u16FSW)
5693	{
5694	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5695	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5696	iemFpuUpdateFSWOnly(pCtx, u16FSW);
5697	iemFpuMaybePopOne(pCtx);
5698	iemFpuMaybePopOne(pCtx);
5699	}
5700
5701
5702	/**
5703	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
5704	*
5705	* @param pIemCpu The IEM per CPU data.
5706	* @param u16FSW The FSW from the current instruction.
5707	* @param iEffSeg The effective memory operand selector register.
5708	* @param GCPtrEff The effective memory operand offset.
5709	*/
5710	static void iemFpuUpdateFSWWithMemOpThenPop(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5711	{
5712	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5713	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5714	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5715	iemFpuUpdateFSWOnly(pCtx, u16FSW);
5716	iemFpuMaybePopOne(pCtx);
5717	}
5718
5719
5720	/**
5721	* Worker routine for raising an FPU stack underflow exception.
5722	*
5723	* @param pIemCpu The IEM per CPU data.
5724	* @param iStReg The stack register being accessed.
5725	* @param pCtx The CPU context.
5726	*/
5727	static void iemFpuStackUnderflowOnly(PIEMCPU pIemCpu, uint8_t iStReg, PCPUMCTX pCtx)
5728	{
5729	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
5730	if (pCtx->fpu.FCW & X86_FCW_IM)
5731	{
5732	/* Masked underflow. */
5733	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5734	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
5735	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
5736	if (iStReg != UINT8_MAX)
5737	{
5738	pCtx->fpu.FTW \|= RT_BIT(iReg);
5739	iemFpuStoreQNan(&pCtx->fpu.aRegs[iStReg].r80);
5740	}
5741	}
5742	else
5743	{
5744	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5745	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5746	}
5747	}
5748
5749
5750	/**
5751	* Raises a FPU stack underflow exception.
5752	*
5753	* @param pIemCpu The IEM per CPU data.
5754	* @param iStReg The destination register that should be loaded
5755	* with QNaN if \#IS is not masked. Specify
5756	* UINT8_MAX if none (like for fcom).
5757	*/
5758	DECL_NO_INLINE(static, void) iemFpuStackUnderflow(PIEMCPU pIemCpu, uint8_t iStReg)
5759	{
5760	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5761	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5762	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
5763	}
5764
5765
5766	DECL_NO_INLINE(static, void)
5767	iemFpuStackUnderflowWithMemOp(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5768	{
5769	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5770	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5771	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5772	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
5773	}
5774
5775
5776	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPop(PIEMCPU pIemCpu, uint8_t iStReg)
5777	{
5778	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5779	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5780	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
5781	iemFpuMaybePopOne(pCtx);
5782	}
5783
5784
5785	DECL_NO_INLINE(static, void)
5786	iemFpuStackUnderflowWithMemOpThenPop(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5787	{
5788	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5789	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5790	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5791	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
5792	iemFpuMaybePopOne(pCtx);
5793	}
5794
5795
5796	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPopPop(PIEMCPU pIemCpu)
5797	{
5798	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5799	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5800	iemFpuStackUnderflowOnly(pIemCpu, UINT8_MAX, pCtx);
5801	iemFpuMaybePopOne(pCtx);
5802	iemFpuMaybePopOne(pCtx);
5803	}
5804
5805
5806	DECL_NO_INLINE(static, void)
5807	iemFpuStackPushUnderflow(PIEMCPU pIemCpu)
5808	{
5809	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5810	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5811
5812	if (pCtx->fpu.FCW & X86_FCW_IM)
5813	{
5814	/* Masked overflow - Push QNaN. */
5815	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
5816	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5817	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
5818	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5819	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
5820	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
5821	iemFpuRotateStackPush(pCtx);
5822	}
5823	else
5824	{
5825	/* Exception pending - don't change TOP or the register stack. */
5826	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5827	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5828	}
5829	}
5830
5831
5832	DECL_NO_INLINE(static, void)
5833	iemFpuStackPushUnderflowTwo(PIEMCPU pIemCpu)
5834	{
5835	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5836	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5837
5838	if (pCtx->fpu.FCW & X86_FCW_IM)
5839	{
5840	/* Masked overflow - Push QNaN. */
5841	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
5842	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5843	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
5844	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5845	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
5846	iemFpuStoreQNan(&pCtx->fpu.aRegs[0].r80);
5847	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
5848	iemFpuRotateStackPush(pCtx);
5849	}
5850	else
5851	{
5852	/* Exception pending - don't change TOP or the register stack. */
5853	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5854	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5855	}
5856	}
5857
5858
5859	/**
5860	* Worker routine for raising an FPU stack overflow exception on a push.
5861	*
5862	* @param pIemCpu The IEM per CPU data.
5863	* @param pCtx The CPU context.
5864	*/
5865	static void iemFpuStackPushOverflowOnly(PIEMCPU pIemCpu, PCPUMCTX pCtx)
5866	{
5867	if (pCtx->fpu.FCW & X86_FCW_IM)
5868	{
5869	/* Masked overflow. */
5870	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
5871	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5872	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
5873	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5874	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
5875	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
5876	iemFpuRotateStackPush(pCtx);
5877	}
5878	else
5879	{
5880	/* Exception pending - don't change TOP or the register stack. */
5881	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
5882	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5883	}
5884	}
5885
5886
5887	/**
5888	* Raises a FPU stack overflow exception on a push.
5889	*
5890	* @param pIemCpu The IEM per CPU data.
5891	*/
5892	DECL_NO_INLINE(static, void) iemFpuStackPushOverflow(PIEMCPU pIemCpu)
5893	{
5894	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5895	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5896	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
5897	}
5898
5899
5900	/**
5901	* Raises a FPU stack overflow exception on a push with a memory operand.
5902	*
5903	* @param pIemCpu The IEM per CPU data.
5904	* @param iEffSeg The effective memory operand selector register.
5905	* @param GCPtrEff The effective memory operand offset.
5906	*/
5907	DECL_NO_INLINE(static, void)
5908	iemFpuStackPushOverflowWithMemOp(PIEMCPU pIemCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
5909	{
5910	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5911	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
5912	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
5913	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
5914	}
5915
5916
5917	static int iemFpuStRegNotEmpty(PIEMCPU pIemCpu, uint8_t iStReg)
5918	{
5919	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5920	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
5921	if (pCtx->fpu.FTW & RT_BIT(iReg))
5922	return VINF_SUCCESS;
5923	return VERR_NOT_FOUND;
5924	}
5925
5926
5927	static int iemFpuStRegNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
5928	{
5929	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5930	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
5931	if (pCtx->fpu.FTW & RT_BIT(iReg))
5932	{
5933	*ppRef = &pCtx->fpu.aRegs[iStReg].r80;
5934	return VINF_SUCCESS;
5935	}
5936	return VERR_NOT_FOUND;
5937	}
5938
5939
5940	static int iemFpu2StRegsNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
5941	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
5942	{
5943	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5944	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
5945	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
5946	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
5947	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
5948	{
5949	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
5950	*ppRef1 = &pCtx->fpu.aRegs[iStReg1].r80;
5951	return VINF_SUCCESS;
5952	}
5953	return VERR_NOT_FOUND;
5954	}
5955
5956
5957	static int iemFpu2StRegsNotEmptyRefFirst(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
5958	{
5959	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5960	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
5961	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
5962	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
5963	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
5964	{
5965	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
5966	return VINF_SUCCESS;
5967	}
5968	return VERR_NOT_FOUND;
5969	}
5970
5971
5972	/**
5973	* Updates the FPU exception status after FCW is changed.
5974	*
5975	* @param pCtx The CPU context.
5976	*/
5977	static void iemFpuRecalcExceptionStatus(PCPUMCTX pCtx)
5978	{
5979	uint16_t u16Fsw = pCtx->fpu.FSW;
5980	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pCtx->fpu.FCW & X86_FCW_XCPT_MASK))
5981	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
5982	else
5983	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
5984	pCtx->fpu.FSW = u16Fsw;
5985	}
5986
5987
5988	/**
5989	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
5990	*
5991	* @returns The full FTW.
5992	* @param pCtx The CPU state.
5993	*/
5994	static uint16_t iemFpuCalcFullFtw(PCCPUMCTX pCtx)
5995	{
5996	uint8_t const u8Ftw = (uint8_t)pCtx->fpu.FTW;
5997	uint16_t u16Ftw = 0;
5998	unsigned const iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
5999	for (unsigned iSt = 0; iSt < 8; iSt++)
6000	{
6001	unsigned const iReg = (iSt + iTop) & 7;
6002	if (!(u8Ftw & RT_BIT(iReg)))
6003	u16Ftw \|= 3 << (iReg * 2); /* empty */
6004	else
6005	{
6006	uint16_t uTag;
6007	PCRTFLOAT80U const pr80Reg = &pCtx->fpu.aRegs[iSt].r80;
6008	if (pr80Reg->s.uExponent == 0x7fff)
6009	uTag = 2; /* Exponent is all 1's => Special. */
6010	else if (pr80Reg->s.uExponent == 0x0000)
6011	{
6012	if (pr80Reg->s.u64Mantissa == 0x0000)
6013	uTag = 1; /* All bits are zero => Zero. */
6014	else
6015	uTag = 2; /* Must be special. */
6016	}
6017	else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
6018	uTag = 0; /* Valid. */
6019	else
6020	uTag = 2; /* Must be special. */
6021
6022	u16Ftw \|= uTag << (iReg * 2); /* empty */
6023	}
6024	}
6025
6026	return u16Ftw;
6027	}
6028
6029
6030	/**
6031	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
6032	*
6033	* @returns The compressed FTW.
6034	* @param u16FullFtw The full FTW to convert.
6035	*/
6036	static uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
6037	{
6038	uint8_t u8Ftw = 0;
6039	for (unsigned i = 0; i < 8; i++)
6040	{
6041	if ((u16FullFtw & 3) != 3 /empty/)
6042	u8Ftw \|= RT_BIT(i);
6043	u16FullFtw >>= 2;
6044	}
6045
6046	return u8Ftw;
6047	}
6048
6049	/** @} */
6050
6051
6052	/** @name Memory access.
6053	*
6054	* @{
6055	*/
6056
6057
6058	/**
6059	* Updates the IEMCPU::cbWritten counter if applicable.
6060	*
6061	* @param pIemCpu The IEM per CPU data.
6062	* @param fAccess The access being accounted for.
6063	* @param cbMem The access size.
6064	*/
6065	DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PIEMCPU pIemCpu, uint32_t fAccess, size_t cbMem)
6066	{
6067	if ( (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK \| IEM_ACCESS_TYPE_WRITE)
6068	\|\| (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_TYPE_WRITE) )
6069	pIemCpu->cbWritten += (uint32_t)cbMem;
6070	}
6071
6072
6073	/**
6074	* Checks if the given segment can be written to, raise the appropriate
6075	* exception if not.
6076	*
6077	* @returns VBox strict status code.
6078	*
6079	* @param pIemCpu The IEM per CPU data.
6080	* @param pHid Pointer to the hidden register.
6081	* @param iSegReg The register number.
6082	* @param pu64BaseAddr Where to return the base address to use for the
6083	* segment. (In 64-bit code it may differ from the
6084	* base in the hidden segment.)
6085	*/
6086	static VBOXSTRICTRC iemMemSegCheckWriteAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
6087	{
6088	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
6089	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
6090	else
6091	{
6092	if (!pHid->Attr.n.u1Present)
6093	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
6094
6095	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
6096	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
6097	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
6098	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_W);
6099	*pu64BaseAddr = pHid->u64Base;
6100	}
6101	return VINF_SUCCESS;
6102	}
6103
6104
6105	/**
6106	* Checks if the given segment can be read from, raise the appropriate
6107	* exception if not.
6108	*
6109	* @returns VBox strict status code.
6110	*
6111	* @param pIemCpu The IEM per CPU data.
6112	* @param pHid Pointer to the hidden register.
6113	* @param iSegReg The register number.
6114	* @param pu64BaseAddr Where to return the base address to use for the
6115	* segment. (In 64-bit code it may differ from the
6116	* base in the hidden segment.)
6117	*/
6118	static VBOXSTRICTRC iemMemSegCheckReadAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
6119	{
6120	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
6121	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
6122	else
6123	{
6124	if (!pHid->Attr.n.u1Present)
6125	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
6126
6127	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
6128	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_R);
6129	*pu64BaseAddr = pHid->u64Base;
6130	}
6131	return VINF_SUCCESS;
6132	}
6133
6134
6135	/**
6136	* Applies the segment limit, base and attributes.
6137	*
6138	* This may raise a \#GP or \#SS.
6139	*
6140	* @returns VBox strict status code.
6141	*
6142	* @param pIemCpu The IEM per CPU data.
6143	* @param fAccess The kind of access which is being performed.
6144	* @param iSegReg The index of the segment register to apply.
6145	* This is UINT8_MAX if none (for IDT, GDT, LDT,
6146	* TSS, ++).
6147	* @param pGCPtrMem Pointer to the guest memory address to apply
6148	* segmentation to. Input and output parameter.
6149	*/
6150	static VBOXSTRICTRC iemMemApplySegment(PIEMCPU pIemCpu, uint32_t fAccess, uint8_t iSegReg,
6151	size_t cbMem, PRTGCPTR pGCPtrMem)
6152	{
6153	if (iSegReg == UINT8_MAX)
6154	return VINF_SUCCESS;
6155
6156	PCPUMSELREGHID pSel = iemSRegGetHid(pIemCpu, iSegReg);
6157	switch (pIemCpu->enmCpuMode)
6158	{
6159	case IEMMODE_16BIT:
6160	case IEMMODE_32BIT:
6161	{
6162	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
6163	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
6164
6165	Assert(pSel->Attr.n.u1Present);
6166	Assert(pSel->Attr.n.u1DescType);
6167	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6168	{
6169	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
6170	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
6171	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
6172
6173	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
6174	{
6175	/** @todo CPL check. */
6176	}
6177
6178	/*
6179	* There are two kinds of data selectors, normal and expand down.
6180	*/
6181	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6182	{
6183	if ( GCPtrFirst32 > pSel->u32Limit
6184	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
6185	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
6186	}
6187	else
6188	{
6189	/*
6190	* The upper boundary is defined by the B bit, not the G bit!
6191	*/
6192	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6193	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6194	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
6195	}
6196	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
6197	}
6198	else
6199	{
6200
6201	/*
6202	* Code selector and usually be used to read thru, writing is
6203	* only permitted in real and V8086 mode.
6204	*/
6205	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
6206	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
6207	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
6208	&& !IEM_IS_REAL_OR_V86_MODE(pIemCpu) )
6209	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
6210
6211	if ( GCPtrFirst32 > pSel->u32Limit
6212	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
6213	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
6214
6215	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
6216	{
6217	/** @todo CPL check. */
6218	}
6219
6220	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
6221	}
6222	return VINF_SUCCESS;
6223	}
6224
6225	case IEMMODE_64BIT:
6226	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
6227	*pGCPtrMem += pSel->u64Base;
6228	return VINF_SUCCESS;
6229
6230	default:
6231	AssertFailedReturn(VERR_INTERNAL_ERROR_5);
6232	}
6233	}
6234
6235
6236	/**
6237	* Translates a virtual address to a physical physical address and checks if we
6238	* can access the page as specified.
6239	*
6240	* @param pIemCpu The IEM per CPU data.
6241	* @param GCPtrMem The virtual address.
6242	* @param fAccess The intended access.
6243	* @param pGCPhysMem Where to return the physical address.
6244	*/
6245	static VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PIEMCPU pIemCpu, RTGCPTR GCPtrMem, uint32_t fAccess,
6246	PRTGCPHYS pGCPhysMem)
6247	{
6248	/** @todo Need a different PGM interface here. We're currently using
6249	* generic / REM interfaces. this won't cut it for R0 & RC. */
6250	RTGCPHYS GCPhys;
6251	uint64_t fFlags;
6252	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrMem, &fFlags, &GCPhys);
6253	if (RT_FAILURE(rc))
6254	{
6255	/** @todo Check unassigned memory in unpaged mode. */
6256	/** @todo Reserved bits in page tables. Requires new PGM interface. */
6257	*pGCPhysMem = NIL_RTGCPHYS;
6258	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, rc);
6259	}
6260
6261	/* If the page is writable and does not have the no-exec bit set, all
6262	access is allowed. Otherwise we'll have to check more carefully... */
6263	if ((fFlags & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
6264	{
6265	/* Write to read only memory? */
6266	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
6267	&& !(fFlags & X86_PTE_RW)
6268	&& ( pIemCpu->uCpl != 0
6269	\|\| (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_WP)))
6270	{
6271	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6272	*pGCPhysMem = NIL_RTGCPHYS;
6273	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6274	}
6275
6276	/* Kernel memory accessed by userland? */
6277	if ( !(fFlags & X86_PTE_US)
6278	&& pIemCpu->uCpl == 3
6279	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6280	{
6281	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6282	*pGCPhysMem = NIL_RTGCPHYS;
6283	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6284	}
6285
6286	/* Executing non-executable memory? */
6287	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
6288	&& (fFlags & X86_PTE_PAE_NX)
6289	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) )
6290	{
6291	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
6292	*pGCPhysMem = NIL_RTGCPHYS;
6293	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
6294	VERR_ACCESS_DENIED);
6295	}
6296	}
6297
6298	/*
6299	* Set the dirty / access flags.
6300	* ASSUMES this is set when the address is translated rather than on committ...
6301	*/
6302	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6303	uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6304	if ((fFlags & fAccessedDirty) != fAccessedDirty)
6305	{
6306	int rc2 = PGMGstModifyPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6307	AssertRC(rc2);
6308	}
6309
6310	GCPhys \|= GCPtrMem & PAGE_OFFSET_MASK;
6311	*pGCPhysMem = GCPhys;
6312	return VINF_SUCCESS;
6313	}
6314
6315
6316
6317	/**
6318	* Maps a physical page.
6319	*
6320	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
6321	* @param pIemCpu The IEM per CPU data.
6322	* @param GCPhysMem The physical address.
6323	* @param fAccess The intended access.
6324	* @param ppvMem Where to return the mapping address.
6325	* @param pLock The PGM lock.
6326	*/
6327	static int iemMemPageMap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
6328	{
6329	#ifdef IEM_VERIFICATION_MODE_FULL
6330	/* Force the alternative path so we can ignore writes. */
6331	if ((fAccess & IEM_ACCESS_TYPE_WRITE) && !pIemCpu->fNoRem)
6332	{
6333	if (IEM_FULL_VERIFICATION_ENABLED(pIemCpu))
6334	{
6335	int rc2 = PGMPhysIemQueryAccess(IEMCPU_TO_VM(pIemCpu), GCPhysMem,
6336	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE), pIemCpu->fBypassHandlers);
6337	if (RT_FAILURE(rc2))
6338	pIemCpu->fProblematicMemory = true;
6339	}
6340	return VERR_PGM_PHYS_TLB_CATCH_ALL;
6341	}
6342	#endif
6343	#ifdef IEM_LOG_MEMORY_WRITES
6344	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6345	return VERR_PGM_PHYS_TLB_CATCH_ALL;
6346	#endif
6347	#ifdef IEM_VERIFICATION_MODE_MINIMAL
6348	return VERR_PGM_PHYS_TLB_CATCH_ALL;
6349	#endif
6350
6351	/** @todo This API may require some improving later. A private deal with PGM
6352	* regarding locking and unlocking needs to be struct. A couple of TLBs
6353	* living in PGM, but with publicly accessible inlined access methods
6354	* could perhaps be an even better solution. */
6355	int rc = PGMPhysIemGCPhys2Ptr(IEMCPU_TO_VM(pIemCpu), IEMCPU_TO_VMCPU(pIemCpu),
6356	GCPhysMem,
6357	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
6358	pIemCpu->fBypassHandlers,
6359	ppvMem,
6360	pLock);
6361	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
6362	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
6363
6364	#ifdef IEM_VERIFICATION_MODE_FULL
6365	if (RT_FAILURE(rc) && IEM_FULL_VERIFICATION_ENABLED(pIemCpu))
6366	pIemCpu->fProblematicMemory = true;
6367	#endif
6368	return rc;
6369	}
6370
6371
6372	/**
6373	* Unmap a page previously mapped by iemMemPageMap.
6374	*
6375	* @param pIemCpu The IEM per CPU data.
6376	* @param GCPhysMem The physical address.
6377	* @param fAccess The intended access.
6378	* @param pvMem What iemMemPageMap returned.
6379	* @param pLock The PGM lock.
6380	*/
6381	DECLINLINE(void) iemMemPageUnmap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
6382	{
6383	NOREF(pIemCpu);
6384	NOREF(GCPhysMem);
6385	NOREF(fAccess);
6386	NOREF(pvMem);
6387	PGMPhysReleasePageMappingLock(IEMCPU_TO_VM(pIemCpu), pLock);
6388	}
6389
6390
6391	/**
6392	* Looks up a memory mapping entry.
6393	*
6394	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
6395	* @param pIemCpu The IEM per CPU data.
6396	* @param pvMem The memory address.
6397	* @param fAccess The access to.
6398	*/
6399	DECLINLINE(int) iemMapLookup(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
6400	{
6401	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
6402	if ( pIemCpu->aMemMappings[0].pv == pvMem
6403	&& (pIemCpu->aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
6404	return 0;
6405	if ( pIemCpu->aMemMappings[1].pv == pvMem
6406	&& (pIemCpu->aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
6407	return 1;
6408	if ( pIemCpu->aMemMappings[2].pv == pvMem
6409	&& (pIemCpu->aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
6410	return 2;
6411	return VERR_NOT_FOUND;
6412	}
6413
6414
6415	/**
6416	* Finds a free memmap entry when using iNextMapping doesn't work.
6417	*
6418	* @returns Memory mapping index, 1024 on failure.
6419	* @param pIemCpu The IEM per CPU data.
6420	*/
6421	static unsigned iemMemMapFindFree(PIEMCPU pIemCpu)
6422	{
6423	/*
6424	* The easy case.
6425	*/
6426	if (pIemCpu->cActiveMappings == 0)
6427	{
6428	pIemCpu->iNextMapping = 1;
6429	return 0;
6430	}
6431
6432	/* There should be enough mappings for all instructions. */
6433	AssertReturn(pIemCpu->cActiveMappings < RT_ELEMENTS(pIemCpu->aMemMappings), 1024);
6434
6435	for (unsigned i = 0; i < RT_ELEMENTS(pIemCpu->aMemMappings); i++)
6436	if (pIemCpu->aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
6437	return i;
6438
6439	AssertFailedReturn(1024);
6440	}
6441
6442
6443	/**
6444	* Commits a bounce buffer that needs writing back and unmaps it.
6445	*
6446	* @returns Strict VBox status code.
6447	* @param pIemCpu The IEM per CPU data.
6448	* @param iMemMap The index of the buffer to commit.
6449	*/
6450	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PIEMCPU pIemCpu, unsigned iMemMap)
6451	{
6452	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
6453	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
6454
6455	/*
6456	* Do the writing.
6457	*/
6458	int rc;
6459	#ifndef IEM_VERIFICATION_MODE_MINIMAL
6460	if ( !pIemCpu->aMemBbMappings[iMemMap].fUnassigned
6461	&& !IEM_VERIFICATION_ENABLED(pIemCpu))
6462	{
6463	uint16_t const cbFirst = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
6464	uint16_t const cbSecond = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
6465	uint8_t const *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
6466	if (!pIemCpu->fBypassHandlers)
6467	{
6468	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
6469	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
6470	pbBuf,
6471	cbFirst);
6472	if (cbSecond && rc == VINF_SUCCESS)
6473	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
6474	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
6475	pbBuf + cbFirst,
6476	cbSecond);
6477	}
6478	else
6479	{
6480	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
6481	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
6482	pbBuf,
6483	cbFirst);
6484	if (cbSecond && rc == VINF_SUCCESS)
6485	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
6486	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
6487	pbBuf + cbFirst,
6488	cbSecond);
6489	}
6490	if (rc != VINF_SUCCESS)
6491	{
6492	/** @todo status code handling */
6493	Log(("iemMemBounceBufferCommitAndUnmap: %s GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
6494	pIemCpu->fBypassHandlers ? "PGMPhysWrite" : "PGMPhysSimpleWriteGCPhys",
6495	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
6496	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
6497	}
6498	}
6499	else
6500	#endif
6501	rc = VINF_SUCCESS;
6502
6503	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
6504	/*
6505	* Record the write(s).
6506	*/
6507	if (!pIemCpu->fNoRem)
6508	{
6509	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6510	if (pEvtRec)
6511	{
6512	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
6513	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst;
6514	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
6515	memcpy(pEvtRec->u.RamWrite.ab, &pIemCpu->aBounceBuffers[iMemMap].ab[0], pIemCpu->aMemBbMappings[iMemMap].cbFirst);
6516	AssertCompile(sizeof(pEvtRec->u.RamWrite.ab) == sizeof(pIemCpu->aBounceBuffers[0].ab));
6517	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6518	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6519	}
6520	if (pIemCpu->aMemBbMappings[iMemMap].cbSecond)
6521	{
6522	pEvtRec = iemVerifyAllocRecord(pIemCpu);
6523	if (pEvtRec)
6524	{
6525	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
6526	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond;
6527	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
6528	memcpy(pEvtRec->u.RamWrite.ab,
6529	&pIemCpu->aBounceBuffers[iMemMap].ab[pIemCpu->aMemBbMappings[iMemMap].cbFirst],
6530	pIemCpu->aMemBbMappings[iMemMap].cbSecond);
6531	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6532	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6533	}
6534	}
6535	}
6536	#endif
6537	#if defined(IEM_VERIFICATION_MODE_MINIMAL) \|\| defined(IEM_LOG_MEMORY_WRITES)
6538	if (rc == VINF_SUCCESS)
6539	{
6540	Log(("IEM Wrote %RGp: %.*Rhxs\n", pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
6541	RT_MAX(RT_MIN(pIemCpu->aMemBbMappings[iMemMap].cbFirst, 64), 1), &pIemCpu->aBounceBuffers[iMemMap].ab[0]));
6542	if (pIemCpu->aMemBbMappings[iMemMap].cbSecond)
6543	Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
6544	RT_MIN(pIemCpu->aMemBbMappings[iMemMap].cbSecond, 64),
6545	&pIemCpu->aBounceBuffers[iMemMap].ab[pIemCpu->aMemBbMappings[iMemMap].cbFirst]));
6546
6547	size_t cbWrote = pIemCpu->aMemBbMappings[iMemMap].cbFirst + pIemCpu->aMemBbMappings[iMemMap].cbSecond;
6548	g_cbIemWrote = cbWrote;
6549	memcpy(g_abIemWrote, &pIemCpu->aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
6550	}
6551	#endif
6552
6553	/*
6554	* Free the mapping entry.
6555	*/
6556	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6557	Assert(pIemCpu->cActiveMappings != 0);
6558	pIemCpu->cActiveMappings--;
6559	return rc;
6560	}
6561
6562
6563	/**
6564	* iemMemMap worker that deals with a request crossing pages.
6565	*/
6566	static VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PIEMCPU pIemCpu, int iMemMap, void **ppvMem,
6567	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
6568	{
6569	/*
6570	* Do the address translations.
6571	*/
6572	RTGCPHYS GCPhysFirst;
6573	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst, fAccess, &GCPhysFirst);
6574	if (rcStrict != VINF_SUCCESS)
6575	return rcStrict;
6576
6577	/** @todo Testcase & AMD-V/VT-x verification: Check if CR2 should really be the
6578	* last byte. */
6579	RTGCPHYS GCPhysSecond;
6580	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst + (cbMem - 1), fAccess, &GCPhysSecond);
6581	if (rcStrict != VINF_SUCCESS)
6582	return rcStrict;
6583	GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
6584
6585	#ifdef IEM_VERIFICATION_MODE_FULL
6586	/*
6587	* Detect problematic memory when verifying so we can select
6588	* the right execution engine. (TLB: Redo this.)
6589	*/
6590	if (IEM_FULL_VERIFICATION_ENABLED(pIemCpu))
6591	{
6592	int rc2 = PGMPhysIemQueryAccess(IEMCPU_TO_VM(pIemCpu), GCPhysFirst,
6593	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE), pIemCpu->fBypassHandlers);
6594	if (RT_SUCCESS(rc2))
6595	rc2 = PGMPhysIemQueryAccess(IEMCPU_TO_VM(pIemCpu), GCPhysSecond,
6596	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE), pIemCpu->fBypassHandlers);
6597	if (RT_FAILURE(rc2))
6598	pIemCpu->fProblematicMemory = true;
6599	}
6600	#endif
6601
6602
6603	/*
6604	* Read in the current memory content if it's a read, execute or partial
6605	* write access.
6606	*/
6607	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
6608	uint32_t const cbFirstPage = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
6609	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
6610
6611	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
6612	{
6613	int rc;
6614	if (!pIemCpu->fBypassHandlers)
6615	{
6616	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbFirstPage);
6617	if (rc != VINF_SUCCESS)
6618	{
6619	/** @todo status code handling */
6620	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
6621	return rc;
6622	}
6623	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage);
6624	if (rc != VINF_SUCCESS)
6625	{
6626	/** @todo status code handling */
6627	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
6628	return rc;
6629	}
6630	}
6631	else
6632	{
6633	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbFirstPage);
6634	if (rc != VINF_SUCCESS)
6635	{
6636	/** @todo status code handling */
6637	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
6638	return rc;
6639	}
6640	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
6641	if (rc != VINF_SUCCESS)
6642	{
6643	/** @todo status code handling */
6644	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
6645	return rc;
6646	}
6647	}
6648
6649	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
6650	if ( !pIemCpu->fNoRem
6651	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
6652	{
6653	/*
6654	* Record the reads.
6655	*/
6656	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6657	if (pEvtRec)
6658	{
6659	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
6660	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
6661	pEvtRec->u.RamRead.cb = cbFirstPage;
6662	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6663	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6664	}
6665	pEvtRec = iemVerifyAllocRecord(pIemCpu);
6666	if (pEvtRec)
6667	{
6668	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
6669	pEvtRec->u.RamRead.GCPhys = GCPhysSecond;
6670	pEvtRec->u.RamRead.cb = cbSecondPage;
6671	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6672	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6673	}
6674	}
6675	#endif
6676	}
6677	#ifdef VBOX_STRICT
6678	else
6679	memset(pbBuf, 0xcc, cbMem);
6680	#endif
6681	#ifdef VBOX_STRICT
6682	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
6683	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
6684	#endif
6685
6686	/*
6687	* Commit the bounce buffer entry.
6688	*/
6689	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
6690	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
6691	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
6692	pIemCpu->aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
6693	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = false;
6694	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
6695	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
6696	pIemCpu->iNextMapping = iMemMap + 1;
6697	pIemCpu->cActiveMappings++;
6698
6699	iemMemUpdateWrittenCounter(pIemCpu, fAccess, cbMem);
6700	*ppvMem = pbBuf;
6701	return VINF_SUCCESS;
6702	}
6703
6704
6705	/**
6706	* iemMemMap woker that deals with iemMemPageMap failures.
6707	*/
6708	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PIEMCPU pIemCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
6709	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
6710	{
6711	/*
6712	* Filter out conditions we can handle and the ones which shouldn't happen.
6713	*/
6714	if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
6715	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
6716	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
6717	{
6718	AssertReturn(RT_FAILURE_NP(rcMap), VERR_INTERNAL_ERROR_3);
6719	return rcMap;
6720	}
6721	pIemCpu->cPotentialExits++;
6722
6723	/*
6724	* Read in the current memory content if it's a read, execute or partial
6725	* write access.
6726	*/
6727	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
6728	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
6729	{
6730	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
6731	memset(pbBuf, 0xff, cbMem);
6732	else
6733	{
6734	int rc;
6735	if (!pIemCpu->fBypassHandlers)
6736	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbMem);
6737	else
6738	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbMem);
6739	if (rc != VINF_SUCCESS)
6740	{
6741	/** @todo status code handling */
6742	Log(("iemMemBounceBufferMapPhys: %s GCPhysFirst=%RGp rc=%Rrc (!!)\n",
6743	pIemCpu->fBypassHandlers ? "PGMPhysRead" : "PGMPhysSimpleReadGCPhys", GCPhysFirst, rc));
6744	return rc;
6745	}
6746	}
6747
6748	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
6749	if ( !pIemCpu->fNoRem
6750	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
6751	{
6752	/*
6753	* Record the read.
6754	*/
6755	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
6756	if (pEvtRec)
6757	{
6758	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
6759	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
6760	pEvtRec->u.RamRead.cb = (uint32_t)cbMem;
6761	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
6762	*pIemCpu->ppIemEvtRecNext = pEvtRec;
6763	}
6764	}
6765	#endif
6766	}
6767	#ifdef VBOX_STRICT
6768	else
6769	memset(pbBuf, 0xcc, cbMem);
6770	#endif
6771	#ifdef VBOX_STRICT
6772	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
6773	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
6774	#endif
6775
6776	/*
6777	* Commit the bounce buffer entry.
6778	*/
6779	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
6780	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
6781	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
6782	pIemCpu->aMemBbMappings[iMemMap].cbSecond = 0;
6783	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
6784	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
6785	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
6786	pIemCpu->iNextMapping = iMemMap + 1;
6787	pIemCpu->cActiveMappings++;
6788
6789	iemMemUpdateWrittenCounter(pIemCpu, fAccess, cbMem);
6790	*ppvMem = pbBuf;
6791	return VINF_SUCCESS;
6792	}
6793
6794
6795
6796	/**
6797	* Maps the specified guest memory for the given kind of access.
6798	*
6799	* This may be using bounce buffering of the memory if it's crossing a page
6800	* boundary or if there is an access handler installed for any of it. Because
6801	* of lock prefix guarantees, we're in for some extra clutter when this
6802	* happens.
6803	*
6804	* This may raise a \#GP, \#SS, \#PF or \#AC.
6805	*
6806	* @returns VBox strict status code.
6807	*
6808	* @param pIemCpu The IEM per CPU data.
6809	* @param ppvMem Where to return the pointer to the mapped
6810	* memory.
6811	* @param cbMem The number of bytes to map. This is usually 1,
6812	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
6813	* string operations it can be up to a page.
6814	* @param iSegReg The index of the segment register to use for
6815	* this access. The base and limits are checked.
6816	* Use UINT8_MAX to indicate that no segmentation
6817	* is required (for IDT, GDT and LDT accesses).
6818	* @param GCPtrMem The address of the guest memory.
6819	* @param a_fAccess How the memory is being accessed. The
6820	* IEM_ACCESS_TYPE_XXX bit is used to figure out
6821	* how to map the memory, while the
6822	* IEM_ACCESS_WHAT_XXX bit is used when raising
6823	* exceptions.
6824	*/
6825	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
6826	{
6827	/*
6828	* Check the input and figure out which mapping entry to use.
6829	*/
6830	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 94); /* 512 is the max! */
6831	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
6832
6833	unsigned iMemMap = pIemCpu->iNextMapping;
6834	if ( iMemMap >= RT_ELEMENTS(pIemCpu->aMemMappings)
6835	\|\| pIemCpu->aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6836	{
6837	iMemMap = iemMemMapFindFree(pIemCpu);
6838	AssertReturn(iMemMap < RT_ELEMENTS(pIemCpu->aMemMappings), VERR_INTERNAL_ERROR_3);
6839	}
6840
6841	/*
6842	* Map the memory, checking that we can actually access it. If something
6843	* slightly complicated happens, fall back on bounce buffering.
6844	*/
6845	VBOXSTRICTRC rcStrict = iemMemApplySegment(pIemCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6846	if (rcStrict != VINF_SUCCESS)
6847	return rcStrict;
6848
6849	if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
6850	return iemMemBounceBufferMapCrossPage(pIemCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
6851
6852	RTGCPHYS GCPhysFirst;
6853	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrMem, fAccess, &GCPhysFirst);
6854	if (rcStrict != VINF_SUCCESS)
6855	return rcStrict;
6856
6857	void *pvMem;
6858	rcStrict = iemMemPageMap(pIemCpu, GCPhysFirst, fAccess, &pvMem, &pIemCpu->aMemMappingLocks[iMemMap].Lock);
6859	if (rcStrict != VINF_SUCCESS)
6860	return iemMemBounceBufferMapPhys(pIemCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6861
6862	/*
6863	* Fill in the mapping table entry.
6864	*/
6865	pIemCpu->aMemMappings[iMemMap].pv = pvMem;
6866	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess;
6867	pIemCpu->iNextMapping = iMemMap + 1;
6868	pIemCpu->cActiveMappings++;
6869
6870	iemMemUpdateWrittenCounter(pIemCpu, fAccess, cbMem);
6871	*ppvMem = pvMem;
6872	return VINF_SUCCESS;
6873	}
6874
6875
6876	/**
6877	* Commits the guest memory if bounce buffered and unmaps it.
6878	*
6879	* @returns Strict VBox status code.
6880	* @param pIemCpu The IEM per CPU data.
6881	* @param pvMem The mapping.
6882	* @param fAccess The kind of access.
6883	*/
6884	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
6885	{
6886	int iMemMap = iemMapLookup(pIemCpu, pvMem, fAccess);
6887	AssertReturn(iMemMap >= 0, iMemMap);
6888
6889	/* If it's bounce buffered, we may need to write back the buffer. */
6890	if (pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6891	{
6892	if (pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6893	return iemMemBounceBufferCommitAndUnmap(pIemCpu, iMemMap);
6894	}
6895	/* Otherwise unlock it. */
6896	else
6897	PGMPhysReleasePageMappingLock(IEMCPU_TO_VM(pIemCpu), &pIemCpu->aMemMappingLocks[iMemMap].Lock);
6898
6899	/* Free the entry. */
6900	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6901	Assert(pIemCpu->cActiveMappings != 0);
6902	pIemCpu->cActiveMappings--;
6903	return VINF_SUCCESS;
6904	}
6905
6906
6907	/**
6908	* Rollbacks mappings, releasing page locks and such.
6909	*
6910	* The caller shall only call this after checking cActiveMappings.
6911	*
6912	* @returns Strict VBox status code to pass up.
6913	* @param pIemCpu The IEM per CPU data.
6914	*/
6915	static void iemMemRollback(PIEMCPU pIemCpu)
6916	{
6917	Assert(pIemCpu->cActiveMappings > 0);
6918
6919	uint32_t iMemMap = RT_ELEMENTS(pIemCpu->aMemMappings);
6920	while (iMemMap-- > 0)
6921	{
6922	uint32_t fAccess = pIemCpu->aMemMappings[iMemMap].fAccess;
6923	if (fAccess != IEM_ACCESS_INVALID)
6924	{
6925	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6926	if (!(fAccess & IEM_ACCESS_BOUNCE_BUFFERED))
6927	PGMPhysReleasePageMappingLock(IEMCPU_TO_VM(pIemCpu), &pIemCpu->aMemMappingLocks[iMemMap].Lock);
6928	Assert(pIemCpu->cActiveMappings > 0);
6929	pIemCpu->cActiveMappings--;
6930	}
6931	}
6932	}
6933
6934
6935	/**
6936	* Fetches a data byte.
6937	*
6938	* @returns Strict VBox status code.
6939	* @param pIemCpu The IEM per CPU data.
6940	* @param pu8Dst Where to return the byte.
6941	* @param iSegReg The index of the segment register to use for
6942	* this access. The base and limits are checked.
6943	* @param GCPtrMem The address of the guest memory.
6944	*/
6945	static VBOXSTRICTRC iemMemFetchDataU8(PIEMCPU pIemCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
6946	{
6947	/* The lazy approach for now... */
6948	uint8_t const *pu8Src;
6949	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
6950	if (rc == VINF_SUCCESS)
6951	{
6952	pu8Dst = pu8Src;
6953	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6954	}
6955	return rc;
6956	}
6957
6958
6959	/**
6960	* Fetches a data word.
6961	*
6962	* @returns Strict VBox status code.
6963	* @param pIemCpu The IEM per CPU data.
6964	* @param pu16Dst Where to return the word.
6965	* @param iSegReg The index of the segment register to use for
6966	* this access. The base and limits are checked.
6967	* @param GCPtrMem The address of the guest memory.
6968	*/
6969	static VBOXSTRICTRC iemMemFetchDataU16(PIEMCPU pIemCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
6970	{
6971	/* The lazy approach for now... */
6972	uint16_t const *pu16Src;
6973	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
6974	if (rc == VINF_SUCCESS)
6975	{
6976	pu16Dst = pu16Src;
6977	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6978	}
6979	return rc;
6980	}
6981
6982
6983	/**
6984	* Fetches a data dword.
6985	*
6986	* @returns Strict VBox status code.
6987	* @param pIemCpu The IEM per CPU data.
6988	* @param pu32Dst Where to return the dword.
6989	* @param iSegReg The index of the segment register to use for
6990	* this access. The base and limits are checked.
6991	* @param GCPtrMem The address of the guest memory.
6992	*/
6993	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
6994	{
6995	/* The lazy approach for now... */
6996	uint32_t const *pu32Src;
6997	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
6998	if (rc == VINF_SUCCESS)
6999	{
7000	pu32Dst = pu32Src;
7001	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
7002	}
7003	return rc;
7004	}
7005
7006
7007	#ifdef SOME_UNUSED_FUNCTION
7008	/**
7009	* Fetches a data dword and sign extends it to a qword.
7010	*
7011	* @returns Strict VBox status code.
7012	* @param pIemCpu The IEM per CPU data.
7013	* @param pu64Dst Where to return the sign extended value.
7014	* @param iSegReg The index of the segment register to use for
7015	* this access. The base and limits are checked.
7016	* @param GCPtrMem The address of the guest memory.
7017	*/
7018	static VBOXSTRICTRC iemMemFetchDataS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7019	{
7020	/* The lazy approach for now... */
7021	int32_t const *pi32Src;
7022	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
7023	if (rc == VINF_SUCCESS)
7024	{
7025	pu64Dst = pi32Src;
7026	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
7027	}
7028	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
7029	else
7030	*pu64Dst = 0;
7031	#endif
7032	return rc;
7033	}
7034	#endif
7035
7036
7037	/**
7038	* Fetches a data qword.
7039	*
7040	* @returns Strict VBox status code.
7041	* @param pIemCpu The IEM per CPU data.
7042	* @param pu64Dst Where to return the qword.
7043	* @param iSegReg The index of the segment register to use for
7044	* this access. The base and limits are checked.
7045	* @param GCPtrMem The address of the guest memory.
7046	*/
7047	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7048	{
7049	/* The lazy approach for now... */
7050	uint64_t const *pu64Src;
7051	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
7052	if (rc == VINF_SUCCESS)
7053	{
7054	pu64Dst = pu64Src;
7055	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
7056	}
7057	return rc;
7058	}
7059
7060
7061	/**
7062	* Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
7063	*
7064	* @returns Strict VBox status code.
7065	* @param pIemCpu The IEM per CPU data.
7066	* @param pu64Dst Where to return the qword.
7067	* @param iSegReg The index of the segment register to use for
7068	* this access. The base and limits are checked.
7069	* @param GCPtrMem The address of the guest memory.
7070	*/
7071	static VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7072	{
7073	/* The lazy approach for now... */
7074	/** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
7075	if (RT_UNLIKELY(GCPtrMem & 15))
7076	return iemRaiseGeneralProtectionFault0(pIemCpu);
7077
7078	uint64_t const *pu64Src;
7079	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
7080	if (rc == VINF_SUCCESS)
7081	{
7082	pu64Dst = pu64Src;
7083	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
7084	}
7085	return rc;
7086	}
7087
7088
7089	/**
7090	* Fetches a data tword.
7091	*
7092	* @returns Strict VBox status code.
7093	* @param pIemCpu The IEM per CPU data.
7094	* @param pr80Dst Where to return the tword.
7095	* @param iSegReg The index of the segment register to use for
7096	* this access. The base and limits are checked.
7097	* @param GCPtrMem The address of the guest memory.
7098	*/
7099	static VBOXSTRICTRC iemMemFetchDataR80(PIEMCPU pIemCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7100	{
7101	/* The lazy approach for now... */
7102	PCRTFLOAT80U pr80Src;
7103	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pr80Src, sizeof(pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
7104	if (rc == VINF_SUCCESS)
7105	{
7106	pr80Dst = pr80Src;
7107	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7108	}
7109	return rc;
7110	}
7111
7112
7113	/**
7114	* Fetches a data dqword (double qword), generally SSE related.
7115	*
7116	* @returns Strict VBox status code.
7117	* @param pIemCpu The IEM per CPU data.
7118	* @param pu128Dst Where to return the qword.
7119	* @param iSegReg The index of the segment register to use for
7120	* this access. The base and limits are checked.
7121	* @param GCPtrMem The address of the guest memory.
7122	*/
7123	static VBOXSTRICTRC iemMemFetchDataU128(PIEMCPU pIemCpu, uint128_t *pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7124	{
7125	/* The lazy approach for now... */
7126	uint128_t const *pu128Src;
7127	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
7128	if (rc == VINF_SUCCESS)
7129	{
7130	pu128Dst = pu128Src;
7131	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7132	}
7133	return rc;
7134	}
7135
7136
7137	/**
7138	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7139	* related.
7140	*
7141	* Raises \#GP(0) if not aligned.
7142	*
7143	* @returns Strict VBox status code.
7144	* @param pIemCpu The IEM per CPU data.
7145	* @param pu128Dst Where to return the qword.
7146	* @param iSegReg The index of the segment register to use for
7147	* this access. The base and limits are checked.
7148	* @param GCPtrMem The address of the guest memory.
7149	*/
7150	static VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PIEMCPU pIemCpu, uint128_t *pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7151	{
7152	/* The lazy approach for now... */
7153	/** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
7154	if ((GCPtrMem & 15) && !(pIemCpu->CTX_SUFF(pCtx)->fpu.MXCSR & X86_MSXCR_MM)) /** @todo should probably check this after applying seg.u64Base... Check real HW. */
7155	return iemRaiseGeneralProtectionFault0(pIemCpu);
7156
7157	uint128_t const *pu128Src;
7158	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
7159	if (rc == VINF_SUCCESS)
7160	{
7161	pu128Dst = pu128Src;
7162	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7163	}
7164	return rc;
7165	}
7166
7167
7168
7169
7170	/**
7171	* Fetches a descriptor register (lgdt, lidt).
7172	*
7173	* @returns Strict VBox status code.
7174	* @param pIemCpu The IEM per CPU data.
7175	* @param pcbLimit Where to return the limit.
7176	* @param pGCPTrBase Where to return the base.
7177	* @param iSegReg The index of the segment register to use for
7178	* this access. The base and limits are checked.
7179	* @param GCPtrMem The address of the guest memory.
7180	* @param enmOpSize The effective operand size.
7181	*/
7182	static VBOXSTRICTRC iemMemFetchDataXdtr(PIEMCPU pIemCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase,
7183	uint8_t iSegReg, RTGCPTR GCPtrMem, IEMMODE enmOpSize)
7184	{
7185	uint8_t const *pu8Src;
7186	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu,
7187	(void **)&pu8Src,
7188	enmOpSize == IEMMODE_64BIT
7189	? 2 + 8
7190	: enmOpSize == IEMMODE_32BIT
7191	? 2 + 4
7192	: 2 + 3,
7193	iSegReg,
7194	GCPtrMem,
7195	IEM_ACCESS_DATA_R);
7196	if (rcStrict == VINF_SUCCESS)
7197	{
7198	*pcbLimit = RT_MAKE_U16(pu8Src[0], pu8Src[1]);
7199	switch (enmOpSize)
7200	{
7201	case IEMMODE_16BIT:
7202	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], 0);
7203	break;
7204	case IEMMODE_32BIT:
7205	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5]);
7206	break;
7207	case IEMMODE_64BIT:
7208	*pGCPtrBase = RT_MAKE_U64_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5],
7209	pu8Src[6], pu8Src[7], pu8Src[8], pu8Src[9]);
7210	break;
7211
7212	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7213	}
7214	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
7215	}
7216	return rcStrict;
7217	}
7218
7219
7220
7221	/**
7222	* Stores a data byte.
7223	*
7224	* @returns Strict VBox status code.
7225	* @param pIemCpu The IEM per CPU data.
7226	* @param iSegReg The index of the segment register to use for
7227	* this access. The base and limits are checked.
7228	* @param GCPtrMem The address of the guest memory.
7229	* @param u8Value The value to store.
7230	*/
7231	static VBOXSTRICTRC iemMemStoreDataU8(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
7232	{
7233	/* The lazy approach for now... */
7234	uint8_t *pu8Dst;
7235	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
7236	if (rc == VINF_SUCCESS)
7237	{
7238	*pu8Dst = u8Value;
7239	rc = iemMemCommitAndUnmap(pIemCpu, pu8Dst, IEM_ACCESS_DATA_W);
7240	}
7241	return rc;
7242	}
7243
7244
7245	/**
7246	* Stores a data word.
7247	*
7248	* @returns Strict VBox status code.
7249	* @param pIemCpu The IEM per CPU data.
7250	* @param iSegReg The index of the segment register to use for
7251	* this access. The base and limits are checked.
7252	* @param GCPtrMem The address of the guest memory.
7253	* @param u16Value The value to store.
7254	*/
7255	static VBOXSTRICTRC iemMemStoreDataU16(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
7256	{
7257	/* The lazy approach for now... */
7258	uint16_t *pu16Dst;
7259	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
7260	if (rc == VINF_SUCCESS)
7261	{
7262	*pu16Dst = u16Value;
7263	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_DATA_W);
7264	}
7265	return rc;
7266	}
7267
7268
7269	/**
7270	* Stores a data dword.
7271	*
7272	* @returns Strict VBox status code.
7273	* @param pIemCpu The IEM per CPU data.
7274	* @param iSegReg The index of the segment register to use for
7275	* this access. The base and limits are checked.
7276	* @param GCPtrMem The address of the guest memory.
7277	* @param u32Value The value to store.
7278	*/
7279	static VBOXSTRICTRC iemMemStoreDataU32(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
7280	{
7281	/* The lazy approach for now... */
7282	uint32_t *pu32Dst;
7283	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
7284	if (rc == VINF_SUCCESS)
7285	{
7286	*pu32Dst = u32Value;
7287	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_DATA_W);
7288	}
7289	return rc;
7290	}
7291
7292
7293	/**
7294	* Stores a data qword.
7295	*
7296	* @returns Strict VBox status code.
7297	* @param pIemCpu The IEM per CPU data.
7298	* @param iSegReg The index of the segment register to use for
7299	* this access. The base and limits are checked.
7300	* @param GCPtrMem The address of the guest memory.
7301	* @param u64Value The value to store.
7302	*/
7303	static VBOXSTRICTRC iemMemStoreDataU64(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
7304	{
7305	/* The lazy approach for now... */
7306	uint64_t *pu64Dst;
7307	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
7308	if (rc == VINF_SUCCESS)
7309	{
7310	*pu64Dst = u64Value;
7311	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_DATA_W);
7312	}
7313	return rc;
7314	}
7315
7316
7317	/**
7318	* Stores a data dqword.
7319	*
7320	* @returns Strict VBox status code.
7321	* @param pIemCpu The IEM per CPU data.
7322	* @param iSegReg The index of the segment register to use for
7323	* this access. The base and limits are checked.
7324	* @param GCPtrMem The address of the guest memory.
7325	* @param u64Value The value to store.
7326	*/
7327	static VBOXSTRICTRC iemMemStoreDataU128(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint128_t u128Value)
7328	{
7329	/* The lazy approach for now... */
7330	uint128_t *pu128Dst;
7331	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
7332	if (rc == VINF_SUCCESS)
7333	{
7334	*pu128Dst = u128Value;
7335	rc = iemMemCommitAndUnmap(pIemCpu, pu128Dst, IEM_ACCESS_DATA_W);
7336	}
7337	return rc;
7338	}
7339
7340
7341	/**
7342	* Stores a data dqword, SSE aligned.
7343	*
7344	* @returns Strict VBox status code.
7345	* @param pIemCpu The IEM per CPU data.
7346	* @param iSegReg The index of the segment register to use for
7347	* this access. The base and limits are checked.
7348	* @param GCPtrMem The address of the guest memory.
7349	* @param u64Value The value to store.
7350	*/
7351	static VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint128_t u128Value)
7352	{
7353	/* The lazy approach for now... */
7354	if ((GCPtrMem & 15) && !(pIemCpu->CTX_SUFF(pCtx)->fpu.MXCSR & X86_MSXCR_MM)) /** @todo should probably check this after applying seg.u64Base... Check real HW. */
7355	return iemRaiseGeneralProtectionFault0(pIemCpu);
7356
7357	uint128_t *pu128Dst;
7358	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
7359	if (rc == VINF_SUCCESS)
7360	{
7361	*pu128Dst = u128Value;
7362	rc = iemMemCommitAndUnmap(pIemCpu, pu128Dst, IEM_ACCESS_DATA_W);
7363	}
7364	return rc;
7365	}
7366
7367
7368	/**
7369	* Stores a descriptor register (sgdt, sidt).
7370	*
7371	* @returns Strict VBox status code.
7372	* @param pIemCpu The IEM per CPU data.
7373	* @param cbLimit The limit.
7374	* @param GCPTrBase The base address.
7375	* @param iSegReg The index of the segment register to use for
7376	* this access. The base and limits are checked.
7377	* @param GCPtrMem The address of the guest memory.
7378	* @param enmOpSize The effective operand size.
7379	*/
7380	static VBOXSTRICTRC iemMemStoreDataXdtr(PIEMCPU pIemCpu, uint16_t cbLimit, RTGCPTR GCPtrBase,
7381	uint8_t iSegReg, RTGCPTR GCPtrMem, IEMMODE enmOpSize)
7382	{
7383	uint8_t *pu8Src;
7384	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu,
7385	(void **)&pu8Src,
7386	enmOpSize == IEMMODE_64BIT
7387	? 2 + 8
7388	: enmOpSize == IEMMODE_32BIT
7389	? 2 + 4
7390	: 2 + 3,
7391	iSegReg,
7392	GCPtrMem,
7393	IEM_ACCESS_DATA_W);
7394	if (rcStrict == VINF_SUCCESS)
7395	{
7396	pu8Src[0] = RT_BYTE1(cbLimit);
7397	pu8Src[1] = RT_BYTE2(cbLimit);
7398	pu8Src[2] = RT_BYTE1(GCPtrBase);
7399	pu8Src[3] = RT_BYTE2(GCPtrBase);
7400	pu8Src[4] = RT_BYTE3(GCPtrBase);
7401	if (enmOpSize == IEMMODE_16BIT)
7402	pu8Src[5] = 0; /* Note! the 286 stored 0xff here. */
7403	else
7404	{
7405	pu8Src[5] = RT_BYTE4(GCPtrBase);
7406	if (enmOpSize == IEMMODE_64BIT)
7407	{
7408	pu8Src[6] = RT_BYTE5(GCPtrBase);
7409	pu8Src[7] = RT_BYTE6(GCPtrBase);
7410	pu8Src[8] = RT_BYTE7(GCPtrBase);
7411	pu8Src[9] = RT_BYTE8(GCPtrBase);
7412	}
7413	}
7414	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_W);
7415	}
7416	return rcStrict;
7417	}
7418
7419
7420	/**
7421	* Pushes a word onto the stack.
7422	*
7423	* @returns Strict VBox status code.
7424	* @param pIemCpu The IEM per CPU data.
7425	* @param u16Value The value to push.
7426	*/
7427	static VBOXSTRICTRC iemMemStackPushU16(PIEMCPU pIemCpu, uint16_t u16Value)
7428	{
7429	/* Increment the stack pointer. */
7430	uint64_t uNewRsp;
7431	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7432	RTGCPTR GCPtrTop = iemRegGetRspForPush(pIemCpu, pCtx, 2, &uNewRsp);
7433
7434	/* Write the word the lazy way. */
7435	uint16_t *pu16Dst;
7436	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7437	if (rc == VINF_SUCCESS)
7438	{
7439	*pu16Dst = u16Value;
7440	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
7441	}
7442
7443	/* Commit the new RSP value unless we an access handler made trouble. */
7444	if (rc == VINF_SUCCESS)
7445	pCtx->rsp = uNewRsp;
7446
7447	return rc;
7448	}
7449
7450
7451	/**
7452	* Pushes a dword onto the stack.
7453	*
7454	* @returns Strict VBox status code.
7455	* @param pIemCpu The IEM per CPU data.
7456	* @param u32Value The value to push.
7457	*/
7458	static VBOXSTRICTRC iemMemStackPushU32(PIEMCPU pIemCpu, uint32_t u32Value)
7459	{
7460	/* Increment the stack pointer. */
7461	uint64_t uNewRsp;
7462	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7463	RTGCPTR GCPtrTop = iemRegGetRspForPush(pIemCpu, pCtx, 4, &uNewRsp);
7464
7465	/* Write the dword the lazy way. */
7466	uint32_t *pu32Dst;
7467	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7468	if (rc == VINF_SUCCESS)
7469	{
7470	*pu32Dst = u32Value;
7471	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
7472	}
7473
7474	/* Commit the new RSP value unless we an access handler made trouble. */
7475	if (rc == VINF_SUCCESS)
7476	pCtx->rsp = uNewRsp;
7477
7478	return rc;
7479	}
7480
7481
7482	/**
7483	* Pushes a dword segment register value onto the stack.
7484	*
7485	* @returns Strict VBox status code.
7486	* @param pIemCpu The IEM per CPU data.
7487	* @param u16Value The value to push.
7488	*/
7489	static VBOXSTRICTRC iemMemStackPushU32SReg(PIEMCPU pIemCpu, uint32_t u32Value)
7490	{
7491	/* Increment the stack pointer. */
7492	uint64_t uNewRsp;
7493	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7494	RTGCPTR GCPtrTop = iemRegGetRspForPush(pIemCpu, pCtx, 4, &uNewRsp);
7495
7496	VBOXSTRICTRC rc;
7497	if (IEM_FULL_VERIFICATION_REM_ENABLED(pIemCpu))
7498	{
7499	/* The recompiler writes a full dword. */
7500	uint32_t *pu32Dst;
7501	rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7502	if (rc == VINF_SUCCESS)
7503	{
7504	*pu32Dst = u32Value;
7505	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
7506	}
7507	}
7508	else
7509	{
7510	/* The intel docs talks about zero extending the selector register
7511	value. My actual intel CPU here might be zero extending the value
7512	but it still only writes the lower word... */
7513	/** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
7514	* happens when crossing an electric page boundrary, is the high word
7515	* checked for write accessibility or not? Probably it is. What about
7516	* segment limits? */
7517	uint16_t *pu16Dst;
7518	rc = iemMemMap(pIemCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_RW);
7519	if (rc == VINF_SUCCESS)
7520	{
7521	*pu16Dst = (uint16_t)u32Value;
7522	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_RW);
7523	}
7524	}
7525
7526	/* Commit the new RSP value unless we an access handler made trouble. */
7527	if (rc == VINF_SUCCESS)
7528	pCtx->rsp = uNewRsp;
7529
7530	return rc;
7531	}
7532
7533
7534	/**
7535	* Pushes a qword onto the stack.
7536	*
7537	* @returns Strict VBox status code.
7538	* @param pIemCpu The IEM per CPU data.
7539	* @param u64Value The value to push.
7540	*/
7541	static VBOXSTRICTRC iemMemStackPushU64(PIEMCPU pIemCpu, uint64_t u64Value)
7542	{
7543	/* Increment the stack pointer. */
7544	uint64_t uNewRsp;
7545	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7546	RTGCPTR GCPtrTop = iemRegGetRspForPush(pIemCpu, pCtx, 8, &uNewRsp);
7547
7548	/* Write the word the lazy way. */
7549	uint64_t *pu64Dst;
7550	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7551	if (rc == VINF_SUCCESS)
7552	{
7553	*pu64Dst = u64Value;
7554	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
7555	}
7556
7557	/* Commit the new RSP value unless we an access handler made trouble. */
7558	if (rc == VINF_SUCCESS)
7559	pCtx->rsp = uNewRsp;
7560
7561	return rc;
7562	}
7563
7564
7565	/**
7566	* Pops a word from the stack.
7567	*
7568	* @returns Strict VBox status code.
7569	* @param pIemCpu The IEM per CPU data.
7570	* @param pu16Value Where to store the popped value.
7571	*/
7572	static VBOXSTRICTRC iemMemStackPopU16(PIEMCPU pIemCpu, uint16_t *pu16Value)
7573	{
7574	/* Increment the stack pointer. */
7575	uint64_t uNewRsp;
7576	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7577	RTGCPTR GCPtrTop = iemRegGetRspForPop(pIemCpu, pCtx, 2, &uNewRsp);
7578
7579	/* Write the word the lazy way. */
7580	uint16_t const *pu16Src;
7581	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7582	if (rc == VINF_SUCCESS)
7583	{
7584	pu16Value = pu16Src;
7585	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
7586
7587	/* Commit the new RSP value. */
7588	if (rc == VINF_SUCCESS)
7589	pCtx->rsp = uNewRsp;
7590	}
7591
7592	return rc;
7593	}
7594
7595
7596	/**
7597	* Pops a dword from the stack.
7598	*
7599	* @returns Strict VBox status code.
7600	* @param pIemCpu The IEM per CPU data.
7601	* @param pu32Value Where to store the popped value.
7602	*/
7603	static VBOXSTRICTRC iemMemStackPopU32(PIEMCPU pIemCpu, uint32_t *pu32Value)
7604	{
7605	/* Increment the stack pointer. */
7606	uint64_t uNewRsp;
7607	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7608	RTGCPTR GCPtrTop = iemRegGetRspForPop(pIemCpu, pCtx, 4, &uNewRsp);
7609
7610	/* Write the word the lazy way. */
7611	uint32_t const *pu32Src;
7612	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7613	if (rc == VINF_SUCCESS)
7614	{
7615	pu32Value = pu32Src;
7616	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
7617
7618	/* Commit the new RSP value. */
7619	if (rc == VINF_SUCCESS)
7620	pCtx->rsp = uNewRsp;
7621	}
7622
7623	return rc;
7624	}
7625
7626
7627	/**
7628	* Pops a qword from the stack.
7629	*
7630	* @returns Strict VBox status code.
7631	* @param pIemCpu The IEM per CPU data.
7632	* @param pu64Value Where to store the popped value.
7633	*/
7634	static VBOXSTRICTRC iemMemStackPopU64(PIEMCPU pIemCpu, uint64_t *pu64Value)
7635	{
7636	/* Increment the stack pointer. */
7637	uint64_t uNewRsp;
7638	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7639	RTGCPTR GCPtrTop = iemRegGetRspForPop(pIemCpu, pCtx, 8, &uNewRsp);
7640
7641	/* Write the word the lazy way. */
7642	uint64_t const *pu64Src;
7643	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7644	if (rc == VINF_SUCCESS)
7645	{
7646	pu64Value = pu64Src;
7647	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
7648
7649	/* Commit the new RSP value. */
7650	if (rc == VINF_SUCCESS)
7651	pCtx->rsp = uNewRsp;
7652	}
7653
7654	return rc;
7655	}
7656
7657
7658	/**
7659	* Pushes a word onto the stack, using a temporary stack pointer.
7660	*
7661	* @returns Strict VBox status code.
7662	* @param pIemCpu The IEM per CPU data.
7663	* @param u16Value The value to push.
7664	* @param pTmpRsp Pointer to the temporary stack pointer.
7665	*/
7666	static VBOXSTRICTRC iemMemStackPushU16Ex(PIEMCPU pIemCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
7667	{
7668	/* Increment the stack pointer. */
7669	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7670	RTUINT64U NewRsp = *pTmpRsp;
7671	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pIemCpu, pCtx, &NewRsp, 2);
7672
7673	/* Write the word the lazy way. */
7674	uint16_t *pu16Dst;
7675	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7676	if (rc == VINF_SUCCESS)
7677	{
7678	*pu16Dst = u16Value;
7679	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
7680	}
7681
7682	/* Commit the new RSP value unless we an access handler made trouble. */
7683	if (rc == VINF_SUCCESS)
7684	*pTmpRsp = NewRsp;
7685
7686	return rc;
7687	}
7688
7689
7690	/**
7691	* Pushes a dword onto the stack, using a temporary stack pointer.
7692	*
7693	* @returns Strict VBox status code.
7694	* @param pIemCpu The IEM per CPU data.
7695	* @param u32Value The value to push.
7696	* @param pTmpRsp Pointer to the temporary stack pointer.
7697	*/
7698	static VBOXSTRICTRC iemMemStackPushU32Ex(PIEMCPU pIemCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
7699	{
7700	/* Increment the stack pointer. */
7701	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7702	RTUINT64U NewRsp = *pTmpRsp;
7703	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pIemCpu, pCtx, &NewRsp, 4);
7704
7705	/* Write the word the lazy way. */
7706	uint32_t *pu32Dst;
7707	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7708	if (rc == VINF_SUCCESS)
7709	{
7710	*pu32Dst = u32Value;
7711	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
7712	}
7713
7714	/* Commit the new RSP value unless we an access handler made trouble. */
7715	if (rc == VINF_SUCCESS)
7716	*pTmpRsp = NewRsp;
7717
7718	return rc;
7719	}
7720
7721
7722	/**
7723	* Pushes a dword onto the stack, using a temporary stack pointer.
7724	*
7725	* @returns Strict VBox status code.
7726	* @param pIemCpu The IEM per CPU data.
7727	* @param u64Value The value to push.
7728	* @param pTmpRsp Pointer to the temporary stack pointer.
7729	*/
7730	static VBOXSTRICTRC iemMemStackPushU64Ex(PIEMCPU pIemCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
7731	{
7732	/* Increment the stack pointer. */
7733	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7734	RTUINT64U NewRsp = *pTmpRsp;
7735	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pIemCpu, pCtx, &NewRsp, 8);
7736
7737	/* Write the word the lazy way. */
7738	uint64_t *pu64Dst;
7739	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7740	if (rc == VINF_SUCCESS)
7741	{
7742	*pu64Dst = u64Value;
7743	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
7744	}
7745
7746	/* Commit the new RSP value unless we an access handler made trouble. */
7747	if (rc == VINF_SUCCESS)
7748	*pTmpRsp = NewRsp;
7749
7750	return rc;
7751	}
7752
7753
7754	/**
7755	* Pops a word from the stack, using a temporary stack pointer.
7756	*
7757	* @returns Strict VBox status code.
7758	* @param pIemCpu The IEM per CPU data.
7759	* @param pu16Value Where to store the popped value.
7760	* @param pTmpRsp Pointer to the temporary stack pointer.
7761	*/
7762	static VBOXSTRICTRC iemMemStackPopU16Ex(PIEMCPU pIemCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
7763	{
7764	/* Increment the stack pointer. */
7765	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7766	RTUINT64U NewRsp = *pTmpRsp;
7767	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pIemCpu, pCtx, &NewRsp, 2);
7768
7769	/* Write the word the lazy way. */
7770	uint16_t const *pu16Src;
7771	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7772	if (rc == VINF_SUCCESS)
7773	{
7774	pu16Value = pu16Src;
7775	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
7776
7777	/* Commit the new RSP value. */
7778	if (rc == VINF_SUCCESS)
7779	*pTmpRsp = NewRsp;
7780	}
7781
7782	return rc;
7783	}
7784
7785
7786	/**
7787	* Pops a dword from the stack, using a temporary stack pointer.
7788	*
7789	* @returns Strict VBox status code.
7790	* @param pIemCpu The IEM per CPU data.
7791	* @param pu32Value Where to store the popped value.
7792	* @param pTmpRsp Pointer to the temporary stack pointer.
7793	*/
7794	static VBOXSTRICTRC iemMemStackPopU32Ex(PIEMCPU pIemCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
7795	{
7796	/* Increment the stack pointer. */
7797	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7798	RTUINT64U NewRsp = *pTmpRsp;
7799	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pIemCpu, pCtx, &NewRsp, 4);
7800
7801	/* Write the word the lazy way. */
7802	uint32_t const *pu32Src;
7803	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7804	if (rc == VINF_SUCCESS)
7805	{
7806	pu32Value = pu32Src;
7807	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
7808
7809	/* Commit the new RSP value. */
7810	if (rc == VINF_SUCCESS)
7811	*pTmpRsp = NewRsp;
7812	}
7813
7814	return rc;
7815	}
7816
7817
7818	/**
7819	* Pops a qword from the stack, using a temporary stack pointer.
7820	*
7821	* @returns Strict VBox status code.
7822	* @param pIemCpu The IEM per CPU data.
7823	* @param pu64Value Where to store the popped value.
7824	* @param pTmpRsp Pointer to the temporary stack pointer.
7825	*/
7826	static VBOXSTRICTRC iemMemStackPopU64Ex(PIEMCPU pIemCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
7827	{
7828	/* Increment the stack pointer. */
7829	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7830	RTUINT64U NewRsp = *pTmpRsp;
7831	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pIemCpu, pCtx, &NewRsp, 8);
7832
7833	/* Write the word the lazy way. */
7834	uint64_t const *pu64Src;
7835	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7836	if (rcStrict == VINF_SUCCESS)
7837	{
7838	pu64Value = pu64Src;
7839	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
7840
7841	/* Commit the new RSP value. */
7842	if (rcStrict == VINF_SUCCESS)
7843	*pTmpRsp = NewRsp;
7844	}
7845
7846	return rcStrict;
7847	}
7848
7849
7850	/**
7851	* Begin a special stack push (used by interrupt, exceptions and such).
7852	*
7853	* This will raise #SS or #PF if appropriate.
7854	*
7855	* @returns Strict VBox status code.
7856	* @param pIemCpu The IEM per CPU data.
7857	* @param cbMem The number of bytes to push onto the stack.
7858	* @param ppvMem Where to return the pointer to the stack memory.
7859	* As with the other memory functions this could be
7860	* direct access or bounce buffered access, so
7861	* don't commit register until the commit call
7862	* succeeds.
7863	* @param puNewRsp Where to return the new RSP value. This must be
7864	* passed unchanged to
7865	* iemMemStackPushCommitSpecial().
7866	*/
7867	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp)
7868	{
7869	Assert(cbMem < UINT8_MAX);
7870	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7871	RTGCPTR GCPtrTop = iemRegGetRspForPush(pIemCpu, pCtx, (uint8_t)cbMem, puNewRsp);
7872	return iemMemMap(pIemCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
7873	}
7874
7875
7876	/**
7877	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
7878	*
7879	* This will update the rSP.
7880	*
7881	* @returns Strict VBox status code.
7882	* @param pIemCpu The IEM per CPU data.
7883	* @param pvMem The pointer returned by
7884	* iemMemStackPushBeginSpecial().
7885	* @param uNewRsp The new RSP value returned by
7886	* iemMemStackPushBeginSpecial().
7887	*/
7888	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp)
7889	{
7890	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, pvMem, IEM_ACCESS_STACK_W);
7891	if (rcStrict == VINF_SUCCESS)
7892	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
7893	return rcStrict;
7894	}
7895
7896
7897	/**
7898	* Begin a special stack pop (used by iret, retf and such).
7899	*
7900	* This will raise \#SS or \#PF if appropriate.
7901	*
7902	* @returns Strict VBox status code.
7903	* @param pIemCpu The IEM per CPU data.
7904	* @param cbMem The number of bytes to push onto the stack.
7905	* @param ppvMem Where to return the pointer to the stack memory.
7906	* @param puNewRsp Where to return the new RSP value. This must be
7907	* passed unchanged to
7908	* iemMemStackPopCommitSpecial() or applied
7909	* manually if iemMemStackPopDoneSpecial() is used.
7910	*/
7911	static VBOXSTRICTRC iemMemStackPopBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
7912	{
7913	Assert(cbMem < UINT8_MAX);
7914	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7915	RTGCPTR GCPtrTop = iemRegGetRspForPop(pIemCpu, pCtx, (uint8_t)cbMem, puNewRsp);
7916	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7917	}
7918
7919
7920	/**
7921	* Continue a special stack pop (used by iret and retf).
7922	*
7923	* This will raise \#SS or \#PF if appropriate.
7924	*
7925	* @returns Strict VBox status code.
7926	* @param pIemCpu The IEM per CPU data.
7927	* @param cbMem The number of bytes to push onto the stack.
7928	* @param ppvMem Where to return the pointer to the stack memory.
7929	* @param puNewRsp Where to return the new RSP value. This must be
7930	* passed unchanged to
7931	* iemMemStackPopCommitSpecial() or applied
7932	* manually if iemMemStackPopDoneSpecial() is used.
7933	*/
7934	static VBOXSTRICTRC iemMemStackPopContinueSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
7935	{
7936	Assert(cbMem < UINT8_MAX);
7937	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7938	RTUINT64U NewRsp;
7939	NewRsp.u = *puNewRsp;
7940	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pIemCpu, pCtx, &NewRsp, 8);
7941	*puNewRsp = NewRsp.u;
7942	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
7943	}
7944
7945
7946	/**
7947	* Commits a special stack pop (started by iemMemStackPopBeginSpecial).
7948	*
7949	* This will update the rSP.
7950	*
7951	* @returns Strict VBox status code.
7952	* @param pIemCpu The IEM per CPU data.
7953	* @param pvMem The pointer returned by
7954	* iemMemStackPopBeginSpecial().
7955	* @param uNewRsp The new RSP value returned by
7956	* iemMemStackPopBeginSpecial().
7957	*/
7958	static VBOXSTRICTRC iemMemStackPopCommitSpecial(PIEMCPU pIemCpu, void const *pvMem, uint64_t uNewRsp)
7959	{
7960	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
7961	if (rcStrict == VINF_SUCCESS)
7962	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
7963	return rcStrict;
7964	}
7965
7966
7967	/**
7968	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
7969	* iemMemStackPopContinueSpecial).
7970	*
7971	* The caller will manually commit the rSP.
7972	*
7973	* @returns Strict VBox status code.
7974	* @param pIemCpu The IEM per CPU data.
7975	* @param pvMem The pointer returned by
7976	* iemMemStackPopBeginSpecial() or
7977	* iemMemStackPopContinueSpecial().
7978	*/
7979	static VBOXSTRICTRC iemMemStackPopDoneSpecial(PIEMCPU pIemCpu, void const *pvMem)
7980	{
7981	return iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
7982	}
7983
7984
7985	/**
7986	* Fetches a system table byte.
7987	*
7988	* @returns Strict VBox status code.
7989	* @param pIemCpu The IEM per CPU data.
7990	* @param pbDst Where to return the byte.
7991	* @param iSegReg The index of the segment register to use for
7992	* this access. The base and limits are checked.
7993	* @param GCPtrMem The address of the guest memory.
7994	*/
7995	static VBOXSTRICTRC iemMemFetchSysU8(PIEMCPU pIemCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem)
7996	{
7997	/* The lazy approach for now... */
7998	uint8_t const *pbSrc;
7999	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pbSrc, sizeof(pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
8000	if (rc == VINF_SUCCESS)
8001	{
8002	pbDst = pbSrc;
8003	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
8004	}
8005	return rc;
8006	}
8007
8008
8009	/**
8010	* Fetches a system table word.
8011	*
8012	* @returns Strict VBox status code.
8013	* @param pIemCpu The IEM per CPU data.
8014	* @param pu16Dst Where to return the word.
8015	* @param iSegReg The index of the segment register to use for
8016	* this access. The base and limits are checked.
8017	* @param GCPtrMem The address of the guest memory.
8018	*/
8019	static VBOXSTRICTRC iemMemFetchSysU16(PIEMCPU pIemCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
8020	{
8021	/* The lazy approach for now... */
8022	uint16_t const *pu16Src;
8023	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
8024	if (rc == VINF_SUCCESS)
8025	{
8026	pu16Dst = pu16Src;
8027	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
8028	}
8029	return rc;
8030	}
8031
8032
8033	/**
8034	* Fetches a system table dword.
8035	*
8036	* @returns Strict VBox status code.
8037	* @param pIemCpu The IEM per CPU data.
8038	* @param pu32Dst Where to return the dword.
8039	* @param iSegReg The index of the segment register to use for
8040	* this access. The base and limits are checked.
8041	* @param GCPtrMem The address of the guest memory.
8042	*/
8043	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
8044	{
8045	/* The lazy approach for now... */
8046	uint32_t const *pu32Src;
8047	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
8048	if (rc == VINF_SUCCESS)
8049	{
8050	pu32Dst = pu32Src;
8051	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
8052	}
8053	return rc;
8054	}
8055
8056
8057	/**
8058	* Fetches a system table qword.
8059	*
8060	* @returns Strict VBox status code.
8061	* @param pIemCpu The IEM per CPU data.
8062	* @param pu64Dst Where to return the qword.
8063	* @param iSegReg The index of the segment register to use for
8064	* this access. The base and limits are checked.
8065	* @param GCPtrMem The address of the guest memory.
8066	*/
8067	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
8068	{
8069	/* The lazy approach for now... */
8070	uint64_t const *pu64Src;
8071	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
8072	if (rc == VINF_SUCCESS)
8073	{
8074	pu64Dst = pu64Src;
8075	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
8076	}
8077	return rc;
8078	}
8079
8080
8081	/**
8082	* Fetches a descriptor table entry with caller specified error code.
8083	*
8084	* @returns Strict VBox status code.
8085	* @param pIemCpu The IEM per CPU.
8086	* @param pDesc Where to return the descriptor table entry.
8087	* @param uSel The selector which table entry to fetch.
8088	* @param uXcpt The exception to raise on table lookup error.
8089	* @param uErrorCode The error code associated with the exception.
8090	*/
8091	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt,
8092	uint16_t uErrorCode)
8093	{
8094	AssertPtr(pDesc);
8095	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
8096
8097	/** @todo did the 286 require all 8 bytes to be accessible? */
8098	/*
8099	* Get the selector table base and check bounds.
8100	*/
8101	RTGCPTR GCPtrBase;
8102	if (uSel & X86_SEL_LDT)
8103	{
8104	if ( !pCtx->ldtr.Attr.n.u1Present
8105	\|\| (uSel \| X86_SEL_RPL_LDT) > pCtx->ldtr.u32Limit )
8106	{
8107	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
8108	uSel, pCtx->ldtr.u32Limit, pCtx->ldtr.Sel));
8109	return iemRaiseXcptOrInt(pIemCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8110	uErrorCode, 0);
8111	}
8112
8113	Assert(pCtx->ldtr.Attr.n.u1Present);
8114	GCPtrBase = pCtx->ldtr.u64Base;
8115	}
8116	else
8117	{
8118	if ((uSel \| X86_SEL_RPL_LDT) > pCtx->gdtr.cbGdt)
8119	{
8120	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pCtx->gdtr.cbGdt));
8121	return iemRaiseXcptOrInt(pIemCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8122	uErrorCode, 0);
8123	}
8124	GCPtrBase = pCtx->gdtr.pGdt;
8125	}
8126
8127	/*
8128	* Read the legacy descriptor and maybe the long mode extensions if
8129	* required.
8130	*/
8131	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
8132	if (rcStrict == VINF_SUCCESS)
8133	{
8134	if ( !IEM_IS_LONG_MODE(pIemCpu)
8135	\|\| pDesc->Legacy.Gen.u1DescType)
8136	pDesc->Long.au64[1] = 0;
8137	else if ((uint32_t)(uSel \| X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pCtx->ldtr.u32Limit : pCtx->gdtr.cbGdt))
8138	rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel \| X86_SEL_RPL_LDT) + 1);
8139	else
8140	{
8141	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
8142	/** @todo is this the right exception? */
8143	return iemRaiseXcptOrInt(pIemCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
8144	}
8145	}
8146	return rcStrict;
8147	}
8148
8149
8150	/**
8151	* Fetches a descriptor table entry.
8152	*
8153	* @returns Strict VBox status code.
8154	* @param pIemCpu The IEM per CPU.
8155	* @param pDesc Where to return the descriptor table entry.
8156	* @param uSel The selector which table entry to fetch.
8157	* @param uXcpt The exception to raise on table lookup error.
8158	*/
8159	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt)
8160	{
8161	return iemMemFetchSelDescWithErr(pIemCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
8162	}
8163
8164
8165	/**
8166	* Fakes a long mode stack selector for SS = 0.
8167	*
8168	* @param pDescSs Where to return the fake stack descriptor.
8169	* @param uDpl The DPL we want.
8170	*/
8171	static void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
8172	{
8173	pDescSs->Long.au64[0] = 0;
8174	pDescSs->Long.au64[1] = 0;
8175	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
8176	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
8177	pDescSs->Long.Gen.u2Dpl = uDpl;
8178	pDescSs->Long.Gen.u1Present = 1;
8179	pDescSs->Long.Gen.u1Long = 1;
8180	}
8181
8182
8183	/**
8184	* Marks the selector descriptor as accessed (only non-system descriptors).
8185	*
8186	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
8187	* will therefore skip the limit checks.
8188	*
8189	* @returns Strict VBox status code.
8190	* @param pIemCpu The IEM per CPU.
8191	* @param uSel The selector.
8192	*/
8193	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel)
8194	{
8195	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
8196
8197	/*
8198	* Get the selector table base and calculate the entry address.
8199	*/
8200	RTGCPTR GCPtr = uSel & X86_SEL_LDT
8201	? pCtx->ldtr.u64Base
8202	: pCtx->gdtr.pGdt;
8203	GCPtr += uSel & X86_SEL_MASK;
8204
8205	/*
8206	* ASMAtomicBitSet will assert if the address is misaligned, so do some
8207	* ugly stuff to avoid this. This will make sure it's an atomic access
8208	* as well more or less remove any question about 8-bit or 32-bit accesss.
8209	*/
8210	VBOXSTRICTRC rcStrict;
8211	uint32_t volatile *pu32;
8212	if ((GCPtr & 3) == 0)
8213	{
8214	/* The normal case, map the 32-bit bits around the accessed bit (40). */
8215	GCPtr += 2 + 2;
8216	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
8217	if (rcStrict != VINF_SUCCESS)
8218	return rcStrict;
8219	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
8220	}
8221	else
8222	{
8223	/* The misaligned GDT/LDT case, map the whole thing. */
8224	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
8225	if (rcStrict != VINF_SUCCESS)
8226	return rcStrict;
8227	switch ((uintptr_t)pu32 & 3)
8228	{
8229	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
8230	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
8231	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
8232	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
8233	}
8234	}
8235
8236	return iemMemCommitAndUnmap(pIemCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
8237	}
8238
8239	/** @} */
8240
8241
8242	/*
8243	* Include the C/C++ implementation of instruction.
8244	*/
8245	#include "IEMAllCImpl.cpp.h"
8246
8247
8248
8249	/** @name "Microcode" macros.
8250	*
8251	* The idea is that we should be able to use the same code to interpret
8252	* instructions as well as recompiler instructions. Thus this obfuscation.
8253	*
8254	* @{
8255	*/
8256	#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
8257	#define IEM_MC_END() }
8258	#define IEM_MC_PAUSE() do {} while (0)
8259	#define IEM_MC_CONTINUE() do {} while (0)
8260
8261	/** Internal macro. */
8262	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
8263	do \
8264	{ \
8265	VBOXSTRICTRC rcStrict2 = a_Expr; \
8266	if (rcStrict2 != VINF_SUCCESS) \
8267	return rcStrict2; \
8268	} while (0)
8269
8270	#define IEM_MC_ADVANCE_RIP() iemRegUpdateRipAndClearRF(pIemCpu)
8271	#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pIemCpu, a_i8))
8272	#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pIemCpu, a_i16))
8273	#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pIemCpu, a_i32))
8274	#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u16NewIP)))
8275	#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u32NewIP)))
8276	#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u64NewIP)))
8277
8278	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pIemCpu)
8279	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
8280	do { \
8281	if ((pIemCpu)->CTX_SUFF(pCtx)->cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
8282	return iemRaiseDeviceNotAvailable(pIemCpu); \
8283	} while (0)
8284	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
8285	do { \
8286	if ((pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW & X86_FSW_ES) \
8287	return iemRaiseMathFault(pIemCpu); \
8288	} while (0)
8289	#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT() \
8290	do { \
8291	if ( (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_EM) \
8292	\|\| !(pIemCpu->CTX_SUFF(pCtx)->cr4 & X86_CR4_OSFSXR) \
8293	\|\| !IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(X86_CPUID_FEATURE_EDX_SSE2) ) \
8294	return iemRaiseUndefinedOpcode(pIemCpu); \
8295	if (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_TS) \
8296	return iemRaiseDeviceNotAvailable(pIemCpu); \
8297	} while (0)
8298	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
8299	do { \
8300	if ( ((pIemCpu)->CTX_SUFF(pCtx)->cr0 & X86_CR0_EM) \
8301	\|\| !IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(X86_CPUID_FEATURE_EDX_MMX) ) \
8302	return iemRaiseUndefinedOpcode(pIemCpu); \
8303	if (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_TS) \
8304	return iemRaiseDeviceNotAvailable(pIemCpu); \
8305	} while (0)
8306	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() \
8307	do { \
8308	if ( ((pIemCpu)->CTX_SUFF(pCtx)->cr0 & X86_CR0_EM) \
8309	\|\| ( !IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(X86_CPUID_FEATURE_EDX_SSE) \
8310	&& !IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX(X86_CPUID_AMD_FEATURE_EDX_AXMMX) ) ) \
8311	return iemRaiseUndefinedOpcode(pIemCpu); \
8312	if (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_TS) \
8313	return iemRaiseDeviceNotAvailable(pIemCpu); \
8314	} while (0)
8315	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
8316	do { \
8317	if (pIemCpu->uCpl != 0) \
8318	return iemRaiseGeneralProtectionFault0(pIemCpu); \
8319	} while (0)
8320
8321
8322	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
8323	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
8324	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
8325	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
8326	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
8327	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
8328	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
8329	uint32_t a_Name; \
8330	uint32_t *a_pName = &a_Name
8331	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
8332	do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u = (a_EFlags); Assert((pIemCpu)->CTX_SUFF(pCtx)->eflags.u & X86_EFL_1); } while (0)
8333
8334	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
8335	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
8336
8337	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
8338	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
8339	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
8340	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
8341	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
8342	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
8343	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
8344	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
8345	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
8346	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
8347	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
8348	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
8349	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
8350	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
8351	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pIemCpu, (a_iGReg))
8352	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pIemCpu, (a_iGReg))
8353	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
8354	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) (a_u16Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
8355	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) (a_u32Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
8356	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) (a_u64Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
8357	#define IEM_MC_FETCH_CR0_U16(a_u16Dst) (a_u16Dst) = (uint16_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
8358	#define IEM_MC_FETCH_CR0_U32(a_u32Dst) (a_u32Dst) = (uint32_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
8359	#define IEM_MC_FETCH_CR0_U64(a_u64Dst) (a_u64Dst) = (pIemCpu)->CTX_SUFF(pCtx)->cr0
8360	#define IEM_MC_FETCH_LDTR_U16(a_u16Dst) (a_u16Dst) = (pIemCpu)->CTX_SUFF(pCtx)->ldtr.Sel
8361	#define IEM_MC_FETCH_LDTR_U32(a_u32Dst) (a_u32Dst) = (pIemCpu)->CTX_SUFF(pCtx)->ldtr.Sel
8362	#define IEM_MC_FETCH_LDTR_U64(a_u64Dst) (a_u64Dst) = (pIemCpu)->CTX_SUFF(pCtx)->ldtr.Sel
8363	#define IEM_MC_FETCH_TR_U16(a_u16Dst) (a_u16Dst) = (pIemCpu)->CTX_SUFF(pCtx)->tr.Sel
8364	#define IEM_MC_FETCH_TR_U32(a_u32Dst) (a_u32Dst) = (pIemCpu)->CTX_SUFF(pCtx)->tr.Sel
8365	#define IEM_MC_FETCH_TR_U64(a_u64Dst) (a_u64Dst) = (pIemCpu)->CTX_SUFF(pCtx)->tr.Sel
8366	/** @note Not for IOPL or IF testing or modification. */
8367	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = (pIemCpu)->CTX_SUFF(pCtx)->eflags.u
8368	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
8369	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pIemCpu->CTX_SUFF(pCtx)->fpu.FSW
8370	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pIemCpu->CTX_SUFF(pCtx)->fpu.FCW
8371
8372	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8(pIemCpu, (a_iGReg)) = (a_u8Value)
8373	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u16Value)
8374	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
8375	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u64Value)
8376	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
8377	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
8378	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
8379	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
8380	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= UINT32_MAX
8381	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
8382	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
8383	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
8384
8385	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8(pIemCpu, (a_iGReg))
8386	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t *)iemGRegRef(pIemCpu, (a_iGReg))
8387	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
8388	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
8389	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t *)iemGRegRef(pIemCpu, (a_iGReg))
8390	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t *)iemGRegRef(pIemCpu, (a_iGReg))
8391	/** @note Not for IOPL or IF testing or modification. */
8392	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
8393
8394	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u8Value)
8395	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u16Value)
8396	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
8397	do { \
8398	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
8399	*pu32Reg += (a_u32Value); \
8400	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
8401	} while (0)
8402	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u64Value)
8403
8404	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u8Value)
8405	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u16Value)
8406	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
8407	do { \
8408	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
8409	*pu32Reg -= (a_u32Value); \
8410	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
8411	} while (0)
8412	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u64Value)
8413
8414	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pIemCpu, (a_iGReg)); } while (0)
8415	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pIemCpu, (a_iGReg)); } while (0)
8416	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pIemCpu, (a_iGReg)); } while (0)
8417	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pIemCpu, (a_iGReg)); } while (0)
8418	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
8419	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
8420	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
8421
8422	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
8423	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
8424	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
8425	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
8426
8427	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
8428	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
8429	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
8430
8431	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
8432	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
8433
8434	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
8435	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
8436	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
8437
8438	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
8439	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
8440	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
8441
8442	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
8443
8444	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
8445
8446	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u8Value)
8447	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u16Value)
8448	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
8449	do { \
8450	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
8451	*pu32Reg &= (a_u32Value); \
8452	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
8453	} while (0)
8454	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u64Value)
8455
8456	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u8Value)
8457	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u16Value)
8458	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
8459	do { \
8460	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
8461	*pu32Reg \|= (a_u32Value); \
8462	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
8463	} while (0)
8464	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u64Value)
8465
8466
8467	/** @note Not for IOPL or IF modification. */
8468	#define IEM_MC_SET_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u \|= (a_fBit); } while (0)
8469	/** @note Not for IOPL or IF modification. */
8470	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u &= ~(a_fBit); } while (0)
8471	/** @note Not for IOPL or IF modification. */
8472	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u ^= (a_fBit); } while (0)
8473
8474	#define IEM_MC_CLEAR_FSW_EX() do { (pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
8475
8476
8477	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
8478	do { (a_u64Value) = pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].mmx; } while (0)
8479	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
8480	do { (a_u32Value) = pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].au32[0]; } while (0)
8481	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
8482	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].mmx = (a_u64Value); } while (0)
8483	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
8484	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); } while (0)
8485	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) \
8486	(a_pu64Dst) = (&pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].mmx)
8487	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
8488	(a_pu64Dst) = ((uint64_t const *)&pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].mmx)
8489	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
8490	(a_pu32Dst) = ((uint32_t const *)&pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[(a_iMReg)].mmx)
8491
8492	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
8493	do { (a_u128Value) = pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].xmm; } while (0)
8494	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg) \
8495	do { (a_u64Value) = pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au64[0]; } while (0)
8496	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg) \
8497	do { (a_u32Value) = pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au32[0]; } while (0)
8498	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
8499	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].xmm = (a_u128Value); } while (0)
8500	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
8501	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
8502	pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au64[1] = 0; \
8503	} while (0)
8504	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
8505	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
8506	pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au64[1] = 0; \
8507	} while (0)
8508	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
8509	(a_pu128Dst) = (&pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].xmm)
8510	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
8511	(a_pu128Dst) = ((uint128_t const *)&pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].xmm)
8512	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
8513	(a_pu64Dst) = ((uint64_t const *)&pIemCpu->CTX_SUFF(pCtx)->fpu.aXMM[(a_iXReg)].au64[0])
8514
8515	#define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
8516	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
8517	#define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
8518	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
8519	#define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
8520	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
8521
8522	#define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
8523	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
8524	#define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
8525	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
8526	#define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
8527	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
8528
8529	#define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
8530	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
8531	#define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
8532	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
8533	#define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
8534	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
8535
8536	#define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8537	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
8538
8539	#define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8540	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
8541	#define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
8542	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
8543	#define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
8544	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pIemCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
8545
8546	#define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
8547	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
8548	#define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
8549	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
8550	#define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
8551	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pIemCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
8552
8553	#define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
8554	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pIemCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
8555	#define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
8556	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pIemCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
8557
8558
8559
8560	#define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
8561	do { \
8562	uint8_t u8Tmp; \
8563	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
8564	(a_u16Dst) = u8Tmp; \
8565	} while (0)
8566	#define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
8567	do { \
8568	uint8_t u8Tmp; \
8569	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
8570	(a_u32Dst) = u8Tmp; \
8571	} while (0)
8572	#define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8573	do { \
8574	uint8_t u8Tmp; \
8575	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
8576	(a_u64Dst) = u8Tmp; \
8577	} while (0)
8578	#define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
8579	do { \
8580	uint16_t u16Tmp; \
8581	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
8582	(a_u32Dst) = u16Tmp; \
8583	} while (0)
8584	#define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8585	do { \
8586	uint16_t u16Tmp; \
8587	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
8588	(a_u64Dst) = u16Tmp; \
8589	} while (0)
8590	#define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8591	do { \
8592	uint32_t u32Tmp; \
8593	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
8594	(a_u64Dst) = u32Tmp; \
8595	} while (0)
8596
8597	#define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
8598	do { \
8599	uint8_t u8Tmp; \
8600	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
8601	(a_u16Dst) = (int8_t)u8Tmp; \
8602	} while (0)
8603	#define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
8604	do { \
8605	uint8_t u8Tmp; \
8606	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
8607	(a_u32Dst) = (int8_t)u8Tmp; \
8608	} while (0)
8609	#define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8610	do { \
8611	uint8_t u8Tmp; \
8612	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
8613	(a_u64Dst) = (int8_t)u8Tmp; \
8614	} while (0)
8615	#define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
8616	do { \
8617	uint16_t u16Tmp; \
8618	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
8619	(a_u32Dst) = (int16_t)u16Tmp; \
8620	} while (0)
8621	#define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8622	do { \
8623	uint16_t u16Tmp; \
8624	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
8625	(a_u64Dst) = (int16_t)u16Tmp; \
8626	} while (0)
8627	#define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
8628	do { \
8629	uint32_t u32Tmp; \
8630	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
8631	(a_u64Dst) = (int32_t)u32Tmp; \
8632	} while (0)
8633
8634	#define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
8635	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
8636	#define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
8637	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
8638	#define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
8639	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
8640	#define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
8641	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
8642
8643	#define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
8644	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
8645	#define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
8646	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
8647	#define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
8648	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
8649	#define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
8650	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
8651
8652	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
8653	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
8654	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
8655	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
8656	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u32 = UINT32_C(0xffc00000)
8657	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
8658	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
8659	do { \
8660	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
8661	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
8662	} while (0)
8663
8664	#define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
8665	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
8666	#define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
8667	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
8668
8669
8670	#define IEM_MC_PUSH_U16(a_u16Value) \
8671	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pIemCpu, (a_u16Value)))
8672	#define IEM_MC_PUSH_U32(a_u32Value) \
8673	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pIemCpu, (a_u32Value)))
8674	#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
8675	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pIemCpu, (a_u32Value)))
8676	#define IEM_MC_PUSH_U64(a_u64Value) \
8677	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pIemCpu, (a_u64Value)))
8678
8679	#define IEM_MC_POP_U16(a_pu16Value) \
8680	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pIemCpu, (a_pu16Value)))
8681	#define IEM_MC_POP_U32(a_pu32Value) \
8682	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pIemCpu, (a_pu32Value)))
8683	#define IEM_MC_POP_U64(a_pu64Value) \
8684	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pIemCpu, (a_pu64Value)))
8685
8686	/** Maps guest memory for direct or bounce buffered access.
8687	* The purpose is to pass it to an operand implementation, thus the a_iArg.
8688	* @remarks May return.
8689	*/
8690	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
8691	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
8692
8693	/** Maps guest memory for direct or bounce buffered access.
8694	* The purpose is to pass it to an operand implementation, thus the a_iArg.
8695	* @remarks May return.
8696	*/
8697	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
8698	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
8699
8700	/** Commits the memory and unmaps the guest memory.
8701	* @remarks May return.
8702	*/
8703	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
8704	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess)))
8705
8706	/** Commits the memory and unmaps the guest memory unless the FPU status word
8707	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
8708	* that would cause FLD not to store.
8709	*
8710	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
8711	* store, while \#P will not.
8712	*
8713	* @remarks May in theory return - for now.
8714	*/
8715	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
8716	do { \
8717	if ( !(a_u16FSW & X86_FSW_ES) \
8718	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
8719	& ~(pIemCpu->CTX_SUFF(pCtx)->fpu.FCW & X86_FCW_MASK_ALL) ) ) \
8720	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess))); \
8721	} while (0)
8722
8723	/** Calculate efficient address from R/M. */
8724	#define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
8725	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pIemCpu, (bRm), (cbImm), &(a_GCPtrEff)))
8726
8727	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
8728	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
8729	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
8730	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
8731	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
8732	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
8733	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
8734
8735	/**
8736	* Defers the rest of the instruction emulation to a C implementation routine
8737	* and returns, only taking the standard parameters.
8738	*
8739	* @param a_pfnCImpl The pointer to the C routine.
8740	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
8741	*/
8742	#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
8743
8744	/**
8745	* Defers the rest of instruction emulation to a C implementation routine and
8746	* returns, taking one argument in addition to the standard ones.
8747	*
8748	* @param a_pfnCImpl The pointer to the C routine.
8749	* @param a0 The argument.
8750	*/
8751	#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
8752
8753	/**
8754	* Defers the rest of the instruction emulation to a C implementation routine
8755	* and returns, taking two arguments in addition to the standard ones.
8756	*
8757	* @param a_pfnCImpl The pointer to the C routine.
8758	* @param a0 The first extra argument.
8759	* @param a1 The second extra argument.
8760	*/
8761	#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
8762
8763	/**
8764	* Defers the rest of the instruction emulation to a C implementation routine
8765	* and returns, taking three arguments in addition to the standard ones.
8766	*
8767	* @param a_pfnCImpl The pointer to the C routine.
8768	* @param a0 The first extra argument.
8769	* @param a1 The second extra argument.
8770	* @param a2 The third extra argument.
8771	*/
8772	#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
8773
8774	/**
8775	* Defers the rest of the instruction emulation to a C implementation routine
8776	* and returns, taking four arguments in addition to the standard ones.
8777	*
8778	* @param a_pfnCImpl The pointer to the C routine.
8779	* @param a0 The first extra argument.
8780	* @param a1 The second extra argument.
8781	* @param a2 The third extra argument.
8782	* @param a3 The fourth extra argument.
8783	*/
8784	#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2, a3)
8785
8786	/**
8787	* Defers the rest of the instruction emulation to a C implementation routine
8788	* and returns, taking two arguments in addition to the standard ones.
8789	*
8790	* @param a_pfnCImpl The pointer to the C routine.
8791	* @param a0 The first extra argument.
8792	* @param a1 The second extra argument.
8793	* @param a2 The third extra argument.
8794	* @param a3 The fourth extra argument.
8795	* @param a4 The fifth extra argument.
8796	*/
8797	#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2, a3, a4)
8798
8799	/**
8800	* Defers the entire instruction emulation to a C implementation routine and
8801	* returns, only taking the standard parameters.
8802	*
8803	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
8804	*
8805	* @param a_pfnCImpl The pointer to the C routine.
8806	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
8807	*/
8808	#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
8809
8810	/**
8811	* Defers the entire instruction emulation to a C implementation routine and
8812	* returns, taking one argument in addition to the standard ones.
8813	*
8814	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
8815	*
8816	* @param a_pfnCImpl The pointer to the C routine.
8817	* @param a0 The argument.
8818	*/
8819	#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
8820
8821	/**
8822	* Defers the entire instruction emulation to a C implementation routine and
8823	* returns, taking two arguments in addition to the standard ones.
8824	*
8825	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
8826	*
8827	* @param a_pfnCImpl The pointer to the C routine.
8828	* @param a0 The first extra argument.
8829	* @param a1 The second extra argument.
8830	*/
8831	#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
8832
8833	/**
8834	* Defers the entire instruction emulation to a C implementation routine and
8835	* returns, taking three arguments in addition to the standard ones.
8836	*
8837	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
8838	*
8839	* @param a_pfnCImpl The pointer to the C routine.
8840	* @param a0 The first extra argument.
8841	* @param a1 The second extra argument.
8842	* @param a2 The third extra argument.
8843	*/
8844	#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
8845
8846	/**
8847	* Calls a FPU assembly implementation taking one visible argument.
8848	*
8849	* @param a_pfnAImpl Pointer to the assembly FPU routine.
8850	* @param a0 The first extra argument.
8851	*/
8852	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
8853	do { \
8854	iemFpuPrepareUsage(pIemCpu); \
8855	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0)); \
8856	} while (0)
8857
8858	/**
8859	* Calls a FPU assembly implementation taking two visible arguments.
8860	*
8861	* @param a_pfnAImpl Pointer to the assembly FPU routine.
8862	* @param a0 The first extra argument.
8863	* @param a1 The second extra argument.
8864	*/
8865	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
8866	do { \
8867	iemFpuPrepareUsage(pIemCpu); \
8868	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1)); \
8869	} while (0)
8870
8871	/**
8872	* Calls a FPU assembly implementation taking three visible arguments.
8873	*
8874	* @param a_pfnAImpl Pointer to the assembly FPU routine.
8875	* @param a0 The first extra argument.
8876	* @param a1 The second extra argument.
8877	* @param a2 The third extra argument.
8878	*/
8879	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
8880	do { \
8881	iemFpuPrepareUsage(pIemCpu); \
8882	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1), (a2)); \
8883	} while (0)
8884
8885	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
8886	do { \
8887	(a_FpuData).FSW = (a_FSW); \
8888	(a_FpuData).r80Result = *(a_pr80Value); \
8889	} while (0)
8890
8891	/** Pushes FPU result onto the stack. */
8892	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
8893	iemFpuPushResult(pIemCpu, &a_FpuData)
8894	/** Pushes FPU result onto the stack and sets the FPUDP. */
8895	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
8896	iemFpuPushResultWithMemOp(pIemCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
8897
8898	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
8899	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
8900	iemFpuPushResultTwo(pIemCpu, &a_FpuDataTwo)
8901
8902	/** Stores FPU result in a stack register. */
8903	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
8904	iemFpuStoreResult(pIemCpu, &a_FpuData, a_iStReg)
8905	/** Stores FPU result in a stack register and pops the stack. */
8906	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
8907	iemFpuStoreResultThenPop(pIemCpu, &a_FpuData, a_iStReg)
8908	/** Stores FPU result in a stack register and sets the FPUDP. */
8909	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
8910	iemFpuStoreResultWithMemOp(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
8911	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
8912	* stack. */
8913	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
8914	iemFpuStoreResultWithMemOpThenPop(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
8915
8916	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
8917	#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
8918	iemFpuUpdateOpcodeAndIp(pIemCpu)
8919	/** Free a stack register (for FFREE and FFREEP). */
8920	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
8921	iemFpuStackFree(pIemCpu, a_iStReg)
8922	/** Increment the FPU stack pointer. */
8923	#define IEM_MC_FPU_STACK_INC_TOP() \
8924	iemFpuStackIncTop(pIemCpu)
8925	/** Decrement the FPU stack pointer. */
8926	#define IEM_MC_FPU_STACK_DEC_TOP() \
8927	iemFpuStackDecTop(pIemCpu)
8928
8929	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
8930	#define IEM_MC_UPDATE_FSW(a_u16FSW) \
8931	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
8932	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
8933	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
8934	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
8935	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
8936	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
8937	iemFpuUpdateFSWWithMemOp(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
8938	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
8939	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
8940	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
8941	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
8942	* stack. */
8943	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
8944	iemFpuUpdateFSWWithMemOpThenPop(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
8945	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
8946	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
8947	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
8948
8949	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
8950	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
8951	iemFpuStackUnderflow(pIemCpu, a_iStDst)
8952	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
8953	* stack. */
8954	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
8955	iemFpuStackUnderflowThenPop(pIemCpu, a_iStDst)
8956	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
8957	* FPUDS. */
8958	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
8959	iemFpuStackUnderflowWithMemOp(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
8960	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
8961	* FPUDS. Pops stack. */
8962	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
8963	iemFpuStackUnderflowWithMemOpThenPop(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
8964	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
8965	* stack twice. */
8966	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
8967	iemFpuStackUnderflowThenPopPop(pIemCpu)
8968	/** Raises a FPU stack underflow exception for an instruction pushing a result
8969	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
8970	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
8971	iemFpuStackPushUnderflow(pIemCpu)
8972	/** Raises a FPU stack underflow exception for an instruction pushing a result
8973	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
8974	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
8975	iemFpuStackPushUnderflowTwo(pIemCpu)
8976
8977	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
8978	* FPUIP, FPUCS and FOP. */
8979	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
8980	iemFpuStackPushOverflow(pIemCpu)
8981	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
8982	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
8983	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
8984	iemFpuStackPushOverflowWithMemOp(pIemCpu, a_iEffSeg, a_GCPtrEff)
8985	/** Indicates that we (might) have modified the FPU state. */
8986	#define IEM_MC_USED_FPU() \
8987	CPUMSetChangedFlags(IEMCPU_TO_VMCPU(pIemCpu), CPUM_CHANGED_FPU_REM)
8988
8989	/**
8990	* Calls a MMX assembly implementation taking two visible arguments.
8991	*
8992	* @param a_pfnAImpl Pointer to the assembly MMX routine.
8993	* @param a0 The first extra argument.
8994	* @param a1 The second extra argument.
8995	*/
8996	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
8997	do { \
8998	iemFpuPrepareUsage(pIemCpu); \
8999	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1)); \
9000	} while (0)
9001
9002	/**
9003	* Calls a MMX assembly implementation taking three visible arguments.
9004	*
9005	* @param a_pfnAImpl Pointer to the assembly MMX routine.
9006	* @param a0 The first extra argument.
9007	* @param a1 The second extra argument.
9008	* @param a2 The third extra argument.
9009	*/
9010	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
9011	do { \
9012	iemFpuPrepareUsage(pIemCpu); \
9013	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1), (a2)); \
9014	} while (0)
9015
9016
9017	/**
9018	* Calls a SSE assembly implementation taking two visible arguments.
9019	*
9020	* @param a_pfnAImpl Pointer to the assembly MMX routine.
9021	* @param a0 The first extra argument.
9022	* @param a1 The second extra argument.
9023	*/
9024	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
9025	do { \
9026	iemFpuPrepareUsageSse(pIemCpu); \
9027	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1)); \
9028	} while (0)
9029
9030	/**
9031	* Calls a SSE assembly implementation taking three visible arguments.
9032	*
9033	* @param a_pfnAImpl Pointer to the assembly MMX routine.
9034	* @param a0 The first extra argument.
9035	* @param a1 The second extra argument.
9036	* @param a2 The third extra argument.
9037	*/
9038	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
9039	do { \
9040	iemFpuPrepareUsageSse(pIemCpu); \
9041	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1), (a2)); \
9042	} while (0)
9043
9044
9045	/** @note Not for IOPL or IF testing. */
9046	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) {
9047	/** @note Not for IOPL or IF testing. */
9048	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit))) {
9049	/** @note Not for IOPL or IF testing. */
9050	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits)) {
9051	/** @note Not for IOPL or IF testing. */
9052	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits))) {
9053	/** @note Not for IOPL or IF testing. */
9054	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
9055	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
9056	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
9057	/** @note Not for IOPL or IF testing. */
9058	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
9059	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
9060	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
9061	/** @note Not for IOPL or IF testing. */
9062	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
9063	if ( (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
9064	\|\| !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
9065	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
9066	/** @note Not for IOPL or IF testing. */
9067	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
9068	if ( !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
9069	&& !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
9070	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
9071	#define IEM_MC_IF_CX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->cx != 0) {
9072	#define IEM_MC_IF_ECX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->ecx != 0) {
9073	#define IEM_MC_IF_RCX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->rcx != 0) {
9074	/** @note Not for IOPL or IF testing. */
9075	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
9076	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
9077	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
9078	/** @note Not for IOPL or IF testing. */
9079	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
9080	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
9081	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
9082	/** @note Not for IOPL or IF testing. */
9083	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
9084	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
9085	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
9086	/** @note Not for IOPL or IF testing. */
9087	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
9088	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
9089	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
9090	/** @note Not for IOPL or IF testing. */
9091	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
9092	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
9093	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
9094	/** @note Not for IOPL or IF testing. */
9095	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
9096	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
9097	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
9098	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
9099	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if ((uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
9100	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
9101	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) == VINF_SUCCESS) {
9102	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
9103	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) != VINF_SUCCESS) {
9104	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
9105	if (iemFpuStRegNotEmptyRef(pIemCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
9106	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
9107	if (iemFpu2StRegsNotEmptyRef(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
9108	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
9109	if (iemFpu2StRegsNotEmptyRefFirst(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
9110	#define IEM_MC_IF_FCW_IM() \
9111	if (pIemCpu->CTX_SUFF(pCtx)->fpu.FCW & X86_FCW_IM) {
9112
9113	#define IEM_MC_ELSE() } else {
9114	#define IEM_MC_ENDIF() } do {} while (0)
9115
9116	/** @} */
9117
9118
9119	/** @name Opcode Debug Helpers.
9120	* @{
9121	*/
9122	#ifdef DEBUG
9123	# define IEMOP_MNEMONIC(a_szMnemonic) \
9124	Log4(("decode - %04x:%RGv %s%s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs.Sel, pIemCpu->CTX_SUFF(pCtx)->rip, \
9125	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pIemCpu->cInstructions))
9126	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) \
9127	Log4(("decode - %04x:%RGv %s%s %s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs.Sel, pIemCpu->CTX_SUFF(pCtx)->rip, \
9128	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, a_szOps, pIemCpu->cInstructions))
9129	#else
9130	# define IEMOP_MNEMONIC(a_szMnemonic) do { } while (0)
9131	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) do { } while (0)
9132	#endif
9133
9134	/** @} */
9135
9136
9137	/** @name Opcode Helpers.
9138	* @{
9139	*/
9140
9141	/** The instruction raises an \#UD in real and V8086 mode. */
9142	#define IEMOP_HLP_NO_REAL_OR_V86_MODE() \
9143	do \
9144	{ \
9145	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu)) \
9146	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
9147	} while (0)
9148
9149	/** The instruction allows no lock prefixing (in this encoding), throw #UD if
9150	* lock prefixed.
9151	* @deprecated IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX */
9152	#define IEMOP_HLP_NO_LOCK_PREFIX() \
9153	do \
9154	{ \
9155	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
9156	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
9157	} while (0)
9158
9159	/** The instruction is not available in 64-bit mode, throw #UD if we're in
9160	* 64-bit mode. */
9161	#define IEMOP_HLP_NO_64BIT() \
9162	do \
9163	{ \
9164	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
9165	return IEMOP_RAISE_INVALID_OPCODE(); \
9166	} while (0)
9167
9168	/** The instruction is only available in 64-bit mode, throw #UD if we're not in
9169	* 64-bit mode. */
9170	#define IEMOP_HLP_ONLY_64BIT() \
9171	do \
9172	{ \
9173	if (pIemCpu->enmCpuMode != IEMMODE_64BIT) \
9174	return IEMOP_RAISE_INVALID_OPCODE(); \
9175	} while (0)
9176
9177	/** The instruction defaults to 64-bit operand size if 64-bit mode. */
9178	#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
9179	do \
9180	{ \
9181	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
9182	iemRecalEffOpSize64Default(pIemCpu); \
9183	} while (0)
9184
9185	/** The instruction has 64-bit operand size if 64-bit mode. */
9186	#define IEMOP_HLP_64BIT_OP_SIZE() \
9187	do \
9188	{ \
9189	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
9190	pIemCpu->enmEffOpSize = pIemCpu->enmDefOpSize = IEMMODE_64BIT; \
9191	} while (0)
9192
9193	/** Only a REX prefix immediately preceeding the first opcode byte takes
9194	* effect. This macro helps ensuring this as well as logging bad guest code. */
9195	#define IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE(a_szPrf) \
9196	do \
9197	{ \
9198	if (RT_UNLIKELY(pIemCpu->fPrefixes & IEM_OP_PRF_REX)) \
9199	{ \
9200	Log5((a_szPrf ": Overriding REX prefix at %RX16! fPrefixes=%#x\n", \
9201	pIemCpu->CTX_SUFF(pCtx)->rip, pIemCpu->fPrefixes)); \
9202	pIemCpu->fPrefixes &= ~IEM_OP_PRF_REX_MASK; \
9203	pIemCpu->uRexB = 0; \
9204	pIemCpu->uRexIndex = 0; \
9205	pIemCpu->uRexReg = 0; \
9206	iemRecalEffOpSize(pIemCpu); \
9207	} \
9208	} while (0)
9209
9210	/**
9211	* Done decoding.
9212	*/
9213	#define IEMOP_HLP_DONE_DECODING() \
9214	do \
9215	{ \
9216	/nothing for now, maybe later... / \
9217	} while (0)
9218
9219	/**
9220	* Done decoding, raise \#UD exception if lock prefix present.
9221	*/
9222	#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
9223	do \
9224	{ \
9225	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
9226	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
9227	} while (0)
9228	#define IEMOP_HLP_DECODED_NL_1(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_fDisOpType) \
9229	do \
9230	{ \
9231	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
9232	{ \
9233	NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_fDisOpType); \
9234	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
9235	} \
9236	} while (0)
9237	#define IEMOP_HLP_DECODED_NL_2(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_uDisParam1, a_fDisOpType) \
9238	do \
9239	{ \
9240	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
9241	{ \
9242	NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_uDisParam1); NOREF(a_fDisOpType); \
9243	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
9244	} \
9245	} while (0)
9246
9247
9248	/**
9249	* Calculates the effective address of a ModR/M memory operand.
9250	*
9251	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
9252	*
9253	* @return Strict VBox status code.
9254	* @param pIemCpu The IEM per CPU data.
9255	* @param bRm The ModRM byte.
9256	* @param cbImm The size of any immediate following the
9257	* effective address opcode bytes. Important for
9258	* RIP relative addressing.
9259	* @param pGCPtrEff Where to return the effective address.
9260	*/
9261	static VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PIEMCPU pIemCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff)
9262	{
9263	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
9264	PCCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
9265	#define SET_SS_DEF() \
9266	do \
9267	{ \
9268	if (!(pIemCpu->fPrefixes & IEM_OP_PRF_SEG_MASK)) \
9269	pIemCpu->iEffSeg = X86_SREG_SS; \
9270	} while (0)
9271
9272	if (pIemCpu->enmCpuMode != IEMMODE_64BIT)
9273	{
9274	/** @todo Check the effective address size crap! */
9275	if (pIemCpu->enmEffAddrMode == IEMMODE_16BIT)
9276	{
9277	uint16_t u16EffAddr;
9278
9279	/* Handle the disp16 form with no registers first. */
9280	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
9281	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
9282	else
9283	{
9284	/* Get the displacment. */
9285	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9286	{
9287	case 0: u16EffAddr = 0; break;
9288	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
9289	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
9290	default: AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
9291	}
9292
9293	/* Add the base and index registers to the disp. */
9294	switch (bRm & X86_MODRM_RM_MASK)
9295	{
9296	case 0: u16EffAddr += pCtx->bx + pCtx->si; break;
9297	case 1: u16EffAddr += pCtx->bx + pCtx->di; break;
9298	case 2: u16EffAddr += pCtx->bp + pCtx->si; SET_SS_DEF(); break;
9299	case 3: u16EffAddr += pCtx->bp + pCtx->di; SET_SS_DEF(); break;
9300	case 4: u16EffAddr += pCtx->si; break;
9301	case 5: u16EffAddr += pCtx->di; break;
9302	case 6: u16EffAddr += pCtx->bp; SET_SS_DEF(); break;
9303	case 7: u16EffAddr += pCtx->bx; break;
9304	}
9305	}
9306
9307	*pGCPtrEff = u16EffAddr;
9308	}
9309	else
9310	{
9311	Assert(pIemCpu->enmEffAddrMode == IEMMODE_32BIT);
9312	uint32_t u32EffAddr;
9313
9314	/* Handle the disp32 form with no registers first. */
9315	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9316	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
9317	else
9318	{
9319	/* Get the register (or SIB) value. */
9320	switch ((bRm & X86_MODRM_RM_MASK))
9321	{
9322	case 0: u32EffAddr = pCtx->eax; break;
9323	case 1: u32EffAddr = pCtx->ecx; break;
9324	case 2: u32EffAddr = pCtx->edx; break;
9325	case 3: u32EffAddr = pCtx->ebx; break;
9326	case 4: /* SIB */
9327	{
9328	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9329
9330	/* Get the index and scale it. */
9331	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
9332	{
9333	case 0: u32EffAddr = pCtx->eax; break;
9334	case 1: u32EffAddr = pCtx->ecx; break;
9335	case 2: u32EffAddr = pCtx->edx; break;
9336	case 3: u32EffAddr = pCtx->ebx; break;
9337	case 4: u32EffAddr = 0; /none / break;
9338	case 5: u32EffAddr = pCtx->ebp; break;
9339	case 6: u32EffAddr = pCtx->esi; break;
9340	case 7: u32EffAddr = pCtx->edi; break;
9341	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9342	}
9343	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9344
9345	/* add base */
9346	switch (bSib & X86_SIB_BASE_MASK)
9347	{
9348	case 0: u32EffAddr += pCtx->eax; break;
9349	case 1: u32EffAddr += pCtx->ecx; break;
9350	case 2: u32EffAddr += pCtx->edx; break;
9351	case 3: u32EffAddr += pCtx->ebx; break;
9352	case 4: u32EffAddr += pCtx->esp; SET_SS_DEF(); break;
9353	case 5:
9354	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9355	{
9356	u32EffAddr += pCtx->ebp;
9357	SET_SS_DEF();
9358	}
9359	else
9360	{
9361	uint32_t u32Disp;
9362	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9363	u32EffAddr += u32Disp;
9364	}
9365	break;
9366	case 6: u32EffAddr += pCtx->esi; break;
9367	case 7: u32EffAddr += pCtx->edi; break;
9368	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9369	}
9370	break;
9371	}
9372	case 5: u32EffAddr = pCtx->ebp; SET_SS_DEF(); break;
9373	case 6: u32EffAddr = pCtx->esi; break;
9374	case 7: u32EffAddr = pCtx->edi; break;
9375	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9376	}
9377
9378	/* Get and add the displacement. */
9379	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9380	{
9381	case 0:
9382	break;
9383	case 1:
9384	{
9385	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9386	u32EffAddr += i8Disp;
9387	break;
9388	}
9389	case 2:
9390	{
9391	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9392	u32EffAddr += u32Disp;
9393	break;
9394	}
9395	default:
9396	AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
9397	}
9398
9399	}
9400	if (pIemCpu->enmEffAddrMode == IEMMODE_32BIT)
9401	*pGCPtrEff = u32EffAddr;
9402	else
9403	{
9404	Assert(pIemCpu->enmEffAddrMode == IEMMODE_16BIT);
9405	*pGCPtrEff = u32EffAddr & UINT16_MAX;
9406	}
9407	}
9408	}
9409	else
9410	{
9411	uint64_t u64EffAddr;
9412
9413	/* Handle the rip+disp32 form with no registers first. */
9414	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9415	{
9416	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9417	u64EffAddr += pCtx->rip + pIemCpu->offOpcode + cbImm;
9418	}
9419	else
9420	{
9421	/* Get the register (or SIB) value. */
9422	switch ((bRm & X86_MODRM_RM_MASK) \| pIemCpu->uRexB)
9423	{
9424	case 0: u64EffAddr = pCtx->rax; break;
9425	case 1: u64EffAddr = pCtx->rcx; break;
9426	case 2: u64EffAddr = pCtx->rdx; break;
9427	case 3: u64EffAddr = pCtx->rbx; break;
9428	case 5: u64EffAddr = pCtx->rbp; SET_SS_DEF(); break;
9429	case 6: u64EffAddr = pCtx->rsi; break;
9430	case 7: u64EffAddr = pCtx->rdi; break;
9431	case 8: u64EffAddr = pCtx->r8; break;
9432	case 9: u64EffAddr = pCtx->r9; break;
9433	case 10: u64EffAddr = pCtx->r10; break;
9434	case 11: u64EffAddr = pCtx->r11; break;
9435	case 13: u64EffAddr = pCtx->r13; break;
9436	case 14: u64EffAddr = pCtx->r14; break;
9437	case 15: u64EffAddr = pCtx->r15; break;
9438	/* SIB */
9439	case 4:
9440	case 12:
9441	{
9442	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9443
9444	/* Get the index and scale it. */
9445	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pIemCpu->uRexIndex)
9446	{
9447	case 0: u64EffAddr = pCtx->rax; break;
9448	case 1: u64EffAddr = pCtx->rcx; break;
9449	case 2: u64EffAddr = pCtx->rdx; break;
9450	case 3: u64EffAddr = pCtx->rbx; break;
9451	case 4: u64EffAddr = 0; /none / break;
9452	case 5: u64EffAddr = pCtx->rbp; break;
9453	case 6: u64EffAddr = pCtx->rsi; break;
9454	case 7: u64EffAddr = pCtx->rdi; break;
9455	case 8: u64EffAddr = pCtx->r8; break;
9456	case 9: u64EffAddr = pCtx->r9; break;
9457	case 10: u64EffAddr = pCtx->r10; break;
9458	case 11: u64EffAddr = pCtx->r11; break;
9459	case 12: u64EffAddr = pCtx->r12; break;
9460	case 13: u64EffAddr = pCtx->r13; break;
9461	case 14: u64EffAddr = pCtx->r14; break;
9462	case 15: u64EffAddr = pCtx->r15; break;
9463	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9464	}
9465	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9466
9467	/* add base */
9468	switch ((bSib & X86_SIB_BASE_MASK) \| pIemCpu->uRexB)
9469	{
9470	case 0: u64EffAddr += pCtx->rax; break;
9471	case 1: u64EffAddr += pCtx->rcx; break;
9472	case 2: u64EffAddr += pCtx->rdx; break;
9473	case 3: u64EffAddr += pCtx->rbx; break;
9474	case 4: u64EffAddr += pCtx->rsp; SET_SS_DEF(); break;
9475	case 6: u64EffAddr += pCtx->rsi; break;
9476	case 7: u64EffAddr += pCtx->rdi; break;
9477	case 8: u64EffAddr += pCtx->r8; break;
9478	case 9: u64EffAddr += pCtx->r9; break;
9479	case 10: u64EffAddr += pCtx->r10; break;
9480	case 11: u64EffAddr += pCtx->r11; break;
9481	case 12: u64EffAddr += pCtx->r12; break;
9482	case 14: u64EffAddr += pCtx->r14; break;
9483	case 15: u64EffAddr += pCtx->r15; break;
9484	/* complicated encodings */
9485	case 5:
9486	case 13:
9487	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9488	{
9489	if (!pIemCpu->uRexB)
9490	{
9491	u64EffAddr += pCtx->rbp;
9492	SET_SS_DEF();
9493	}
9494	else
9495	u64EffAddr += pCtx->r13;
9496	}
9497	else
9498	{
9499	uint32_t u32Disp;
9500	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9501	u64EffAddr += (int32_t)u32Disp;
9502	}
9503	break;
9504	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9505	}
9506	break;
9507	}
9508	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9509	}
9510
9511	/* Get and add the displacement. */
9512	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9513	{
9514	case 0:
9515	break;
9516	case 1:
9517	{
9518	int8_t i8Disp;
9519	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9520	u64EffAddr += i8Disp;
9521	break;
9522	}
9523	case 2:
9524	{
9525	uint32_t u32Disp;
9526	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9527	u64EffAddr += (int32_t)u32Disp;
9528	break;
9529	}
9530	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
9531	}
9532
9533	}
9534
9535	if (pIemCpu->enmEffAddrMode == IEMMODE_64BIT)
9536	*pGCPtrEff = u64EffAddr;
9537	else
9538	{
9539	Assert(pIemCpu->enmEffAddrMode == IEMMODE_32BIT);
9540	*pGCPtrEff = u64EffAddr & UINT32_MAX;
9541	}
9542	}
9543
9544	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
9545	return VINF_SUCCESS;
9546	}
9547
9548	/** @} */
9549
9550
9551
9552	/*
9553	* Include the instructions
9554	*/
9555	#include "IEMAllInstructions.cpp.h"
9556
9557
9558
9559
9560	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
9561
9562	/**
9563	* Sets up execution verification mode.
9564	*/
9565	static void iemExecVerificationModeSetup(PIEMCPU pIemCpu)
9566	{
9567	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
9568	PCPUMCTX pOrgCtx = pIemCpu->CTX_SUFF(pCtx);
9569
9570	/*
9571	* Always note down the address of the current instruction.
9572	*/
9573	pIemCpu->uOldCs = pOrgCtx->cs.Sel;
9574	pIemCpu->uOldRip = pOrgCtx->rip;
9575
9576	/*
9577	* Enable verification and/or logging.
9578	*/
9579	bool fNewNoRem = !LogIs6Enabled(); /* logging triggers the no-rem/rem verification stuff */;
9580	if ( fNewNoRem
9581	&& ( 0
9582	#if 0 /* auto enable on first paged protected mode interrupt */
9583	\|\| ( pOrgCtx->eflags.Bits.u1IF
9584	&& (pOrgCtx->cr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG)
9585	&& TRPMHasTrap(pVCpu)
9586	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip) )
9587	#endif
9588	#if 0
9589	\|\| ( pOrgCtx->cs == 0x10
9590	&& ( pOrgCtx->rip == 0x90119e3e
9591	\|\| pOrgCtx->rip == 0x901d9810)
9592	#endif
9593	#if 0 /* Auto enable DSL - FPU stuff. */
9594	\|\| ( pOrgCtx->cs == 0x10
9595	&& (// pOrgCtx->rip == 0xc02ec07f
9596	//\|\| pOrgCtx->rip == 0xc02ec082
9597	//\|\| pOrgCtx->rip == 0xc02ec0c9
9598	0
9599	\|\| pOrgCtx->rip == 0x0c010e7c4 /* fxsave */ ) )
9600	#endif
9601	#if 0 /* Auto enable DSL - fstp st0 stuff. */
9602	\|\| (pOrgCtx->cs.Sel == 0x23 pOrgCtx->rip == 0x804aff7)
9603	#endif
9604	#if 0
9605	\|\| pOrgCtx->rip == 0x9022bb3a
9606	#endif
9607	#if 0
9608	\|\| (pOrgCtx->cs.Sel == 0x58 && pOrgCtx->rip == 0x3be) /* NT4SP1 sidt/sgdt in early loader code */
9609	#endif
9610	#if 0
9611	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8013ec28) /* NT4SP1 first str (early boot) */
9612	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x80119e3f) /* NT4SP1 second str (early boot) */
9613	#endif
9614	#if 0 /* NT4SP1 - later on the blue screen, things goes wrong... */
9615	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8010a5df)
9616	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8013a7c4)
9617	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8013a7d2)
9618	#endif
9619	#if 0 /* NT4SP1 - xadd early boot. */
9620	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8019cf0f)
9621	#endif
9622	#if 0 /* NT4SP1 - wrmsr (intel MSR). */
9623	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8011a6d4)
9624	#endif
9625	#if 0 /* NT4SP1 - cmpxchg (AMD). */
9626	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x801684c1)
9627	#endif
9628	#if 0 /* NT4SP1 - fnstsw + 2 (AMD). */
9629	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x801c6b88+2)
9630	#endif
9631	#if 0 /* NT4SP1 - iret to v8086 -- too generic a place? (N/A with GAs installed) */
9632	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8013bd5d)
9633
9634	#endif
9635	#if 0 /* NT4SP1 - iret to v8086 (executing edlin) */
9636	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8013b609)
9637
9638	#endif
9639	#if 0 /* NT4SP1 - frstor [ecx] */
9640	\|\| (pOrgCtx->cs.Sel == 8 && pOrgCtx->rip == 0x8013d11f)
9641	#endif
9642	#if 0 /* xxxxxx - All long mode code. */
9643	\|\| (pOrgCtx->msrEFER & MSR_K6_EFER_LMA)
9644	#endif
9645	#if 0 /* rep movsq linux 3.7 64-bit boot. */
9646	\|\| (pOrgCtx->rip == 0x0000000000100241)
9647	#endif
9648	#if 0 /* linux 3.7 64-bit boot - '000000000215e240'. */
9649	\|\| (pOrgCtx->rip == 0x000000000215e240)
9650	#endif
9651	#if 0 /* DOS's size-overridden iret to v8086. */
9652	\|\| (pOrgCtx->rip == 0x427 && pOrgCtx->cs.Sel == 0xb8)
9653	#endif
9654	)
9655	)
9656	{
9657	RTLogGroupSettings(NULL, "iem.eo.l6.l2");
9658	RTLogFlags(NULL, "enabled");
9659	fNewNoRem = false;
9660	}
9661	if (fNewNoRem != pIemCpu->fNoRem)
9662	{
9663	pIemCpu->fNoRem = fNewNoRem;
9664	if (!fNewNoRem)
9665	{
9666	LogAlways(("Enabling verification mode!\n"));
9667	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
9668	}
9669	else
9670	LogAlways(("Disabling verification mode!\n"));
9671	}
9672
9673	/*
9674	* Switch state.
9675	*/
9676	if (IEM_VERIFICATION_ENABLED(pIemCpu))
9677	{
9678	static CPUMCTX s_DebugCtx; /* Ugly! */
9679
9680	s_DebugCtx = *pOrgCtx;
9681	pIemCpu->CTX_SUFF(pCtx) = &s_DebugCtx;
9682	}
9683
9684	/*
9685	* See if there is an interrupt pending in TRPM and inject it if we can.
9686	*/
9687	pIemCpu->uInjectCpl = UINT8_MAX;
9688	if ( pOrgCtx->eflags.Bits.u1IF
9689	&& TRPMHasTrap(pVCpu)
9690	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
9691	{
9692	uint8_t u8TrapNo;
9693	TRPMEVENT enmType;
9694	RTGCUINT uErrCode;
9695	RTGCPTR uCr2;
9696	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */); AssertRC(rc2);
9697	IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
9698	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
9699	TRPMResetTrap(pVCpu);
9700	pIemCpu->uInjectCpl = pIemCpu->uCpl;
9701	}
9702
9703	/*
9704	* Reset the counters.
9705	*/
9706	pIemCpu->cIOReads = 0;
9707	pIemCpu->cIOWrites = 0;
9708	pIemCpu->fIgnoreRaxRdx = false;
9709	pIemCpu->fOverlappingMovs = false;
9710	pIemCpu->fProblematicMemory = false;
9711	pIemCpu->fUndefinedEFlags = 0;
9712
9713	if (IEM_VERIFICATION_ENABLED(pIemCpu))
9714	{
9715	/*
9716	* Free all verification records.
9717	*/
9718	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pIemEvtRecHead;
9719	pIemCpu->pIemEvtRecHead = NULL;
9720	pIemCpu->ppIemEvtRecNext = &pIemCpu->pIemEvtRecHead;
9721	do
9722	{
9723	while (pEvtRec)
9724	{
9725	PIEMVERIFYEVTREC pNext = pEvtRec->pNext;
9726	pEvtRec->pNext = pIemCpu->pFreeEvtRec;
9727	pIemCpu->pFreeEvtRec = pEvtRec;
9728	pEvtRec = pNext;
9729	}
9730	pEvtRec = pIemCpu->pOtherEvtRecHead;
9731	pIemCpu->pOtherEvtRecHead = NULL;
9732	pIemCpu->ppOtherEvtRecNext = &pIemCpu->pOtherEvtRecHead;
9733	} while (pEvtRec);
9734	}
9735	}
9736
9737
9738	/**
9739	* Allocate an event record.
9740	* @returns Pointer to a record.
9741	*/
9742	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu)
9743	{
9744	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
9745	return NULL;
9746
9747	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pFreeEvtRec;
9748	if (pEvtRec)
9749	pIemCpu->pFreeEvtRec = pEvtRec->pNext;
9750	else
9751	{
9752	if (!pIemCpu->ppIemEvtRecNext)
9753	return NULL; /* Too early (fake PCIBIOS), ignore notification. */
9754
9755	pEvtRec = (PIEMVERIFYEVTREC)MMR3HeapAlloc(IEMCPU_TO_VM(pIemCpu), MM_TAG_EM /* lazy bird/, sizeof(pEvtRec));
9756	if (!pEvtRec)
9757	return NULL;
9758	}
9759	pEvtRec->enmEvent = IEMVERIFYEVENT_INVALID;
9760	pEvtRec->pNext = NULL;
9761	return pEvtRec;
9762	}
9763
9764
9765	/**
9766	* IOMMMIORead notification.
9767	*/
9768	VMM_INT_DECL(void) IEMNotifyMMIORead(PVM pVM, RTGCPHYS GCPhys, size_t cbValue)
9769	{
9770	PVMCPU pVCpu = VMMGetCpu(pVM);
9771	if (!pVCpu)
9772	return;
9773	PIEMCPU pIemCpu = &pVCpu->iem.s;
9774	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
9775	if (!pEvtRec)
9776	return;
9777	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
9778	pEvtRec->u.RamRead.GCPhys = GCPhys;
9779	pEvtRec->u.RamRead.cb = (uint32_t)cbValue;
9780	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
9781	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
9782	}
9783
9784
9785	/**
9786	* IOMMMIOWrite notification.
9787	*/
9788	VMM_INT_DECL(void) IEMNotifyMMIOWrite(PVM pVM, RTGCPHYS GCPhys, uint32_t u32Value, size_t cbValue)
9789	{
9790	PVMCPU pVCpu = VMMGetCpu(pVM);
9791	if (!pVCpu)
9792	return;
9793	PIEMCPU pIemCpu = &pVCpu->iem.s;
9794	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
9795	if (!pEvtRec)
9796	return;
9797	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
9798	pEvtRec->u.RamWrite.GCPhys = GCPhys;
9799	pEvtRec->u.RamWrite.cb = (uint32_t)cbValue;
9800	pEvtRec->u.RamWrite.ab[0] = RT_BYTE1(u32Value);
9801	pEvtRec->u.RamWrite.ab[1] = RT_BYTE2(u32Value);
9802	pEvtRec->u.RamWrite.ab[2] = RT_BYTE3(u32Value);
9803	pEvtRec->u.RamWrite.ab[3] = RT_BYTE4(u32Value);
9804	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
9805	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
9806	}
9807
9808
9809	/**
9810	* IOMIOPortRead notification.
9811	*/
9812	VMM_INT_DECL(void) IEMNotifyIOPortRead(PVM pVM, RTIOPORT Port, size_t cbValue)
9813	{
9814	PVMCPU pVCpu = VMMGetCpu(pVM);
9815	if (!pVCpu)
9816	return;
9817	PIEMCPU pIemCpu = &pVCpu->iem.s;
9818	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
9819	if (!pEvtRec)
9820	return;
9821	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
9822	pEvtRec->u.IOPortRead.Port = Port;
9823	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
9824	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
9825	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
9826	}
9827
9828	/**
9829	* IOMIOPortWrite notification.
9830	*/
9831	VMM_INT_DECL(void) IEMNotifyIOPortWrite(PVM pVM, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
9832	{
9833	PVMCPU pVCpu = VMMGetCpu(pVM);
9834	if (!pVCpu)
9835	return;
9836	PIEMCPU pIemCpu = &pVCpu->iem.s;
9837	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
9838	if (!pEvtRec)
9839	return;
9840	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
9841	pEvtRec->u.IOPortWrite.Port = Port;
9842	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
9843	pEvtRec->u.IOPortWrite.u32Value = u32Value;
9844	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
9845	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
9846	}
9847
9848
9849	VMM_INT_DECL(void) IEMNotifyIOPortReadString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrDst, RTGCUINTREG cTransfers, size_t cbValue)
9850	{
9851	AssertFailed();
9852	}
9853
9854
9855	VMM_INT_DECL(void) IEMNotifyIOPortWriteString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrSrc, RTGCUINTREG cTransfers, size_t cbValue)
9856	{
9857	AssertFailed();
9858	}
9859
9860
9861	/**
9862	* Fakes and records an I/O port read.
9863	*
9864	* @returns VINF_SUCCESS.
9865	* @param pIemCpu The IEM per CPU data.
9866	* @param Port The I/O port.
9867	* @param pu32Value Where to store the fake value.
9868	* @param cbValue The size of the access.
9869	*/
9870	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
9871	{
9872	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
9873	if (pEvtRec)
9874	{
9875	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
9876	pEvtRec->u.IOPortRead.Port = Port;
9877	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
9878	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
9879	*pIemCpu->ppIemEvtRecNext = pEvtRec;
9880	}
9881	pIemCpu->cIOReads++;
9882	*pu32Value = 0xcccccccc;
9883	return VINF_SUCCESS;
9884	}
9885
9886
9887	/**
9888	* Fakes and records an I/O port write.
9889	*
9890	* @returns VINF_SUCCESS.
9891	* @param pIemCpu The IEM per CPU data.
9892	* @param Port The I/O port.
9893	* @param u32Value The value being written.
9894	* @param cbValue The size of the access.
9895	*/
9896	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
9897	{
9898	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
9899	if (pEvtRec)
9900	{
9901	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
9902	pEvtRec->u.IOPortWrite.Port = Port;
9903	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
9904	pEvtRec->u.IOPortWrite.u32Value = u32Value;
9905	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
9906	*pIemCpu->ppIemEvtRecNext = pEvtRec;
9907	}
9908	pIemCpu->cIOWrites++;
9909	return VINF_SUCCESS;
9910	}
9911
9912
9913	/**
9914	* Used to add extra details about a stub case.
9915	* @param pIemCpu The IEM per CPU state.
9916	*/
9917	static void iemVerifyAssertMsg2(PIEMCPU pIemCpu)
9918	{
9919	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
9920	PVM pVM = IEMCPU_TO_VM(pIemCpu);
9921	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
9922	char szRegs[4096];
9923	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
9924	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
9925	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
9926	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
9927	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
9928	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
9929	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
9930	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
9931	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
9932	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
9933	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
9934	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
9935	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
9936	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
9937	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
9938	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
9939	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
9940	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
9941	" efer=%016VR{efer}\n"
9942	" pat=%016VR{pat}\n"
9943	" sf_mask=%016VR{sf_mask}\n"
9944	"krnl_gs_base=%016VR{krnl_gs_base}\n"
9945	" lstar=%016VR{lstar}\n"
9946	" star=%016VR{star} cstar=%016VR{cstar}\n"
9947	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
9948	);
9949
9950	char szInstr1[256];
9951	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, pIemCpu->uOldCs, pIemCpu->uOldRip,
9952	DBGF_DISAS_FLAGS_DEFAULT_MODE,
9953	szInstr1, sizeof(szInstr1), NULL);
9954	char szInstr2[256];
9955	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
9956	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
9957	szInstr2, sizeof(szInstr2), NULL);
9958
9959	RTAssertMsg2Weak("%s%s\n%s\n", szRegs, szInstr1, szInstr2);
9960	}
9961
9962
9963	/**
9964	* Used by iemVerifyAssertRecord and iemVerifyAssertRecords to add a record
9965	* dump to the assertion info.
9966	*
9967	* @param pEvtRec The record to dump.
9968	*/
9969	static void iemVerifyAssertAddRecordDump(PIEMVERIFYEVTREC pEvtRec)
9970	{
9971	switch (pEvtRec->enmEvent)
9972	{
9973	case IEMVERIFYEVENT_IOPORT_READ:
9974	RTAssertMsg2Add("I/O PORT READ from %#6x, %d bytes\n",
9975	pEvtRec->u.IOPortWrite.Port,
9976	pEvtRec->u.IOPortWrite.cbValue);
9977	break;
9978	case IEMVERIFYEVENT_IOPORT_WRITE:
9979	RTAssertMsg2Add("I/O PORT WRITE to %#6x, %d bytes, value %#x\n",
9980	pEvtRec->u.IOPortWrite.Port,
9981	pEvtRec->u.IOPortWrite.cbValue,
9982	pEvtRec->u.IOPortWrite.u32Value);
9983	break;
9984	case IEMVERIFYEVENT_RAM_READ:
9985	RTAssertMsg2Add("RAM READ at %RGp, %#4zx bytes\n",
9986	pEvtRec->u.RamRead.GCPhys,
9987	pEvtRec->u.RamRead.cb);
9988	break;
9989	case IEMVERIFYEVENT_RAM_WRITE:
9990	RTAssertMsg2Add("RAM WRITE at %RGp, %#4zx bytes: %.*Rhxs\n",
9991	pEvtRec->u.RamWrite.GCPhys,
9992	pEvtRec->u.RamWrite.cb,
9993	(int)pEvtRec->u.RamWrite.cb,
9994	pEvtRec->u.RamWrite.ab);
9995	break;
9996	default:
9997	AssertMsgFailed(("Invalid event type %d\n", pEvtRec->enmEvent));
9998	break;
9999	}
10000	}
10001
10002
10003	/**
10004	* Raises an assertion on the specified record, showing the given message with
10005	* a record dump attached.
10006	*
10007	* @param pIemCpu The IEM per CPU data.
10008	* @param pEvtRec1 The first record.
10009	* @param pEvtRec2 The second record.
10010	* @param pszMsg The message explaining why we're asserting.
10011	*/
10012	static void iemVerifyAssertRecords(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec1, PIEMVERIFYEVTREC pEvtRec2, const char *pszMsg)
10013	{
10014	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
10015	iemVerifyAssertAddRecordDump(pEvtRec1);
10016	iemVerifyAssertAddRecordDump(pEvtRec2);
10017	iemVerifyAssertMsg2(pIemCpu);
10018	RTAssertPanic();
10019	}
10020
10021
10022	/**
10023	* Raises an assertion on the specified record, showing the given message with
10024	* a record dump attached.
10025	*
10026	* @param pIemCpu The IEM per CPU data.
10027	* @param pEvtRec1 The first record.
10028	* @param pszMsg The message explaining why we're asserting.
10029	*/
10030	static void iemVerifyAssertRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec, const char *pszMsg)
10031	{
10032	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
10033	iemVerifyAssertAddRecordDump(pEvtRec);
10034	iemVerifyAssertMsg2(pIemCpu);
10035	RTAssertPanic();
10036	}
10037
10038
10039	/**
10040	* Verifies a write record.
10041	*
10042	* @param pIemCpu The IEM per CPU data.
10043	* @param pEvtRec The write record.
10044	* @param fRem Set if REM was doing the other executing. If clear
10045	* it was HM.
10046	*/
10047	static void iemVerifyWriteRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec, bool fRem)
10048	{
10049	uint8_t abBuf[sizeof(pEvtRec->u.RamWrite.ab)]; RT_ZERO(abBuf);
10050	Assert(sizeof(abBuf) >= pEvtRec->u.RamWrite.cb);
10051	int rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), abBuf, pEvtRec->u.RamWrite.GCPhys, pEvtRec->u.RamWrite.cb);
10052	if ( RT_FAILURE(rc)
10053	\|\| memcmp(abBuf, pEvtRec->u.RamWrite.ab, pEvtRec->u.RamWrite.cb) )
10054	{
10055	/* fend off ins */
10056	if ( !pIemCpu->cIOReads
10057	\|\| pEvtRec->u.RamWrite.ab[0] != 0xcc
10058	\|\| ( pEvtRec->u.RamWrite.cb != 1
10059	&& pEvtRec->u.RamWrite.cb != 2
10060	&& pEvtRec->u.RamWrite.cb != 4) )
10061	{
10062	/* fend off ROMs and MMIO */
10063	if ( pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000a0000) > UINT32_C(0x60000)
10064	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0xfffc0000) > UINT32_C(0x40000) )
10065	{
10066	/* fend off fxsave */
10067	if (pEvtRec->u.RamWrite.cb != 512)
10068	{
10069	const char *pszWho = fRem ? "rem" : HMR3IsVmxEnabled(IEMCPU_TO_VM(pIemCpu)->pUVM) ? "vmx" : "svm";
10070	RTAssertMsg1(NULL, __LINE__, __FILE__, __PRETTY_FUNCTION__);
10071	RTAssertMsg2Weak("Memory at %RGv differs\n", pEvtRec->u.RamWrite.GCPhys);
10072	RTAssertMsg2Add("%s: %.*Rhxs\n"
10073	"iem: %.*Rhxs\n",
10074	pszWho, pEvtRec->u.RamWrite.cb, abBuf,
10075	pEvtRec->u.RamWrite.cb, pEvtRec->u.RamWrite.ab);
10076	iemVerifyAssertAddRecordDump(pEvtRec);
10077	iemVerifyAssertMsg2(pIemCpu);
10078	RTAssertPanic();
10079	}
10080	}
10081	}
10082	}
10083
10084	}
10085
10086	/**
10087	* Performs the post-execution verfication checks.
10088	*/
10089	static void iemExecVerificationModeCheck(PIEMCPU pIemCpu)
10090	{
10091	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
10092	return;
10093
10094	/*
10095	* Switch back the state.
10096	*/
10097	PCPUMCTX pOrgCtx = CPUMQueryGuestCtxPtr(IEMCPU_TO_VMCPU(pIemCpu));
10098	PCPUMCTX pDebugCtx = pIemCpu->CTX_SUFF(pCtx);
10099	Assert(pOrgCtx != pDebugCtx);
10100	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
10101
10102	/*
10103	* Execute the instruction in REM.
10104	*/
10105	bool fRem = false;
10106	PVM pVM = IEMCPU_TO_VM(pIemCpu);
10107	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
10108	VBOXSTRICTRC rc = VERR_EM_CANNOT_EXEC_GUEST;
10109	#ifdef IEM_VERIFICATION_MODE_FULL_HM
10110	if ( HMIsEnabled(pVM)
10111	&& pIemCpu->cIOReads == 0
10112	&& pIemCpu->cIOWrites == 0
10113	&& !pIemCpu->fProblematicMemory)
10114	{
10115	unsigned iLoops = 0;
10116	do
10117	{
10118	rc = EMR3HmSingleInstruction(pVM, pVCpu, EM_ONE_INS_FLAGS_RIP_CHANGE);
10119	iLoops++;
10120	} while ( rc == VINF_SUCCESS
10121	\|\| ( rc == VINF_EM_DBG_STEPPED
10122	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10123	&& EMGetInhibitInterruptsPC(pVCpu) == pOrgCtx->rip)
10124	\|\| ( pOrgCtx->rip != pDebugCtx->rip
10125	&& pIemCpu->uInjectCpl != UINT8_MAX
10126	&& iLoops < 8) );
10127	}
10128	#endif
10129	if ( rc == VERR_EM_CANNOT_EXEC_GUEST
10130	\|\| rc == VINF_IOM_R3_IOPORT_READ
10131	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
10132	\|\| rc == VINF_IOM_R3_MMIO_READ
10133	\|\| rc == VINF_IOM_R3_MMIO_READ_WRITE
10134	\|\| rc == VINF_IOM_R3_MMIO_WRITE
10135	)
10136	{
10137	EMRemLock(pVM);
10138	rc = REMR3EmulateInstruction(pVM, pVCpu);
10139	AssertRC(rc);
10140	EMRemUnlock(pVM);
10141	fRem = true;
10142	}
10143
10144	/*
10145	* Compare the register states.
10146	*/
10147	unsigned cDiffs = 0;
10148	if (memcmp(pOrgCtx, pDebugCtx, sizeof(*pDebugCtx)))
10149	{
10150	//Log(("REM and IEM ends up with different registers!\n"));
10151	const char *pszWho = fRem ? "rem" : HMR3IsVmxEnabled(pVM->pUVM) ? "vmx" : "svm";
10152
10153	# define CHECK_FIELD(a_Field) \
10154	do \
10155	{ \
10156	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
10157	{ \
10158	switch (sizeof(pOrgCtx->a_Field)) \
10159	{ \
10160	case 1: RTAssertMsg2Weak(" %8s differs - iem=%02x - %s=%02x\n", #a_Field, pDebugCtx->a_Field, pszWho, pOrgCtx->a_Field); break; \
10161	case 2: RTAssertMsg2Weak(" %8s differs - iem=%04x - %s=%04x\n", #a_Field, pDebugCtx->a_Field, pszWho, pOrgCtx->a_Field); break; \
10162	case 4: RTAssertMsg2Weak(" %8s differs - iem=%08x - %s=%08x\n", #a_Field, pDebugCtx->a_Field, pszWho, pOrgCtx->a_Field); break; \
10163	case 8: RTAssertMsg2Weak(" %8s differs - iem=%016llx - %s=%016llx\n", #a_Field, pDebugCtx->a_Field, pszWho, pOrgCtx->a_Field); break; \
10164	default: RTAssertMsg2Weak(" %8s differs\n", #a_Field); break; \
10165	} \
10166	cDiffs++; \
10167	} \
10168	} while (0)
10169
10170	# define CHECK_BIT_FIELD(a_Field) \
10171	do \
10172	{ \
10173	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
10174	{ \
10175	RTAssertMsg2Weak(" %8s differs - iem=%02x - %s=%02x\n", #a_Field, pDebugCtx->a_Field, pszWho, pOrgCtx->a_Field); \
10176	cDiffs++; \
10177	} \
10178	} while (0)
10179
10180	# define CHECK_SEL(a_Sel) \
10181	do \
10182	{ \
10183	CHECK_FIELD(a_Sel.Sel); \
10184	CHECK_FIELD(a_Sel.Attr.u); \
10185	CHECK_FIELD(a_Sel.u64Base); \
10186	CHECK_FIELD(a_Sel.u32Limit); \
10187	CHECK_FIELD(a_Sel.fFlags); \
10188	} while (0)
10189
10190	#if 1 /* The recompiler doesn't update these the intel way. */
10191	if (fRem)
10192	{
10193	pOrgCtx->fpu.FOP = pDebugCtx->fpu.FOP;
10194	pOrgCtx->fpu.FPUIP = pDebugCtx->fpu.FPUIP;
10195	pOrgCtx->fpu.CS = pDebugCtx->fpu.CS;
10196	pOrgCtx->fpu.Rsrvd1 = pDebugCtx->fpu.Rsrvd1;
10197	pOrgCtx->fpu.FPUDP = pDebugCtx->fpu.FPUDP;
10198	pOrgCtx->fpu.DS = pDebugCtx->fpu.DS;
10199	pOrgCtx->fpu.Rsrvd2 = pDebugCtx->fpu.Rsrvd2;
10200	//pOrgCtx->fpu.MXCSR_MASK = pDebugCtx->fpu.MXCSR_MASK;
10201	if ((pOrgCtx->fpu.FSW & X86_FSW_TOP_MASK) == (pDebugCtx->fpu.FSW & X86_FSW_TOP_MASK))
10202	pOrgCtx->fpu.FSW = pDebugCtx->fpu.FSW;
10203	}
10204	#endif
10205	if (memcmp(&pOrgCtx->fpu, &pDebugCtx->fpu, sizeof(pDebugCtx->fpu)))
10206	{
10207	RTAssertMsg2Weak(" the FPU state differs\n");
10208	cDiffs++;
10209	CHECK_FIELD(fpu.FCW);
10210	CHECK_FIELD(fpu.FSW);
10211	CHECK_FIELD(fpu.FTW);
10212	CHECK_FIELD(fpu.FOP);
10213	CHECK_FIELD(fpu.FPUIP);
10214	CHECK_FIELD(fpu.CS);
10215	CHECK_FIELD(fpu.Rsrvd1);
10216	CHECK_FIELD(fpu.FPUDP);
10217	CHECK_FIELD(fpu.DS);
10218	CHECK_FIELD(fpu.Rsrvd2);
10219	CHECK_FIELD(fpu.MXCSR);
10220	CHECK_FIELD(fpu.MXCSR_MASK);
10221	CHECK_FIELD(fpu.aRegs[0].au64[0]); CHECK_FIELD(fpu.aRegs[0].au64[1]);
10222	CHECK_FIELD(fpu.aRegs[1].au64[0]); CHECK_FIELD(fpu.aRegs[1].au64[1]);
10223	CHECK_FIELD(fpu.aRegs[2].au64[0]); CHECK_FIELD(fpu.aRegs[2].au64[1]);
10224	CHECK_FIELD(fpu.aRegs[3].au64[0]); CHECK_FIELD(fpu.aRegs[3].au64[1]);
10225	CHECK_FIELD(fpu.aRegs[4].au64[0]); CHECK_FIELD(fpu.aRegs[4].au64[1]);
10226	CHECK_FIELD(fpu.aRegs[5].au64[0]); CHECK_FIELD(fpu.aRegs[5].au64[1]);
10227	CHECK_FIELD(fpu.aRegs[6].au64[0]); CHECK_FIELD(fpu.aRegs[6].au64[1]);
10228	CHECK_FIELD(fpu.aRegs[7].au64[0]); CHECK_FIELD(fpu.aRegs[7].au64[1]);
10229	CHECK_FIELD(fpu.aXMM[ 0].au64[0]); CHECK_FIELD(fpu.aXMM[ 0].au64[1]);
10230	CHECK_FIELD(fpu.aXMM[ 1].au64[0]); CHECK_FIELD(fpu.aXMM[ 1].au64[1]);
10231	CHECK_FIELD(fpu.aXMM[ 2].au64[0]); CHECK_FIELD(fpu.aXMM[ 2].au64[1]);
10232	CHECK_FIELD(fpu.aXMM[ 3].au64[0]); CHECK_FIELD(fpu.aXMM[ 3].au64[1]);
10233	CHECK_FIELD(fpu.aXMM[ 4].au64[0]); CHECK_FIELD(fpu.aXMM[ 4].au64[1]);
10234	CHECK_FIELD(fpu.aXMM[ 5].au64[0]); CHECK_FIELD(fpu.aXMM[ 5].au64[1]);
10235	CHECK_FIELD(fpu.aXMM[ 6].au64[0]); CHECK_FIELD(fpu.aXMM[ 6].au64[1]);
10236	CHECK_FIELD(fpu.aXMM[ 7].au64[0]); CHECK_FIELD(fpu.aXMM[ 7].au64[1]);
10237	CHECK_FIELD(fpu.aXMM[ 8].au64[0]); CHECK_FIELD(fpu.aXMM[ 8].au64[1]);
10238	CHECK_FIELD(fpu.aXMM[ 9].au64[0]); CHECK_FIELD(fpu.aXMM[ 9].au64[1]);
10239	CHECK_FIELD(fpu.aXMM[10].au64[0]); CHECK_FIELD(fpu.aXMM[10].au64[1]);
10240	CHECK_FIELD(fpu.aXMM[11].au64[0]); CHECK_FIELD(fpu.aXMM[11].au64[1]);
10241	CHECK_FIELD(fpu.aXMM[12].au64[0]); CHECK_FIELD(fpu.aXMM[12].au64[1]);
10242	CHECK_FIELD(fpu.aXMM[13].au64[0]); CHECK_FIELD(fpu.aXMM[13].au64[1]);
10243	CHECK_FIELD(fpu.aXMM[14].au64[0]); CHECK_FIELD(fpu.aXMM[14].au64[1]);
10244	CHECK_FIELD(fpu.aXMM[15].au64[0]); CHECK_FIELD(fpu.aXMM[15].au64[1]);
10245	for (unsigned i = 0; i < RT_ELEMENTS(pOrgCtx->fpu.au32RsrvdRest); i++)
10246	CHECK_FIELD(fpu.au32RsrvdRest[i]);
10247	}
10248	CHECK_FIELD(rip);
10249	uint32_t fFlagsMask = UINT32_MAX & ~pIemCpu->fUndefinedEFlags;
10250	if ((pOrgCtx->rflags.u & fFlagsMask) != (pDebugCtx->rflags.u & fFlagsMask))
10251	{
10252	RTAssertMsg2Weak(" rflags differs - iem=%08llx %s=%08llx\n", pDebugCtx->rflags.u, pszWho, pOrgCtx->rflags.u);
10253	CHECK_BIT_FIELD(rflags.Bits.u1CF);
10254	CHECK_BIT_FIELD(rflags.Bits.u1Reserved0);
10255	CHECK_BIT_FIELD(rflags.Bits.u1PF);
10256	CHECK_BIT_FIELD(rflags.Bits.u1Reserved1);
10257	CHECK_BIT_FIELD(rflags.Bits.u1AF);
10258	CHECK_BIT_FIELD(rflags.Bits.u1Reserved2);
10259	CHECK_BIT_FIELD(rflags.Bits.u1ZF);
10260	CHECK_BIT_FIELD(rflags.Bits.u1SF);
10261	CHECK_BIT_FIELD(rflags.Bits.u1TF);
10262	CHECK_BIT_FIELD(rflags.Bits.u1IF);
10263	CHECK_BIT_FIELD(rflags.Bits.u1DF);
10264	CHECK_BIT_FIELD(rflags.Bits.u1OF);
10265	CHECK_BIT_FIELD(rflags.Bits.u2IOPL);
10266	CHECK_BIT_FIELD(rflags.Bits.u1NT);
10267	CHECK_BIT_FIELD(rflags.Bits.u1Reserved3);
10268	if (0 && !fRem) /** @todo debug the occational clear RF flags when running against VT-x. */
10269	CHECK_BIT_FIELD(rflags.Bits.u1RF);
10270	CHECK_BIT_FIELD(rflags.Bits.u1VM);
10271	CHECK_BIT_FIELD(rflags.Bits.u1AC);
10272	CHECK_BIT_FIELD(rflags.Bits.u1VIF);
10273	CHECK_BIT_FIELD(rflags.Bits.u1VIP);
10274	CHECK_BIT_FIELD(rflags.Bits.u1ID);
10275	}
10276
10277	if (pIemCpu->cIOReads != 1 && !pIemCpu->fIgnoreRaxRdx)
10278	CHECK_FIELD(rax);
10279	CHECK_FIELD(rcx);
10280	if (!pIemCpu->fIgnoreRaxRdx)
10281	CHECK_FIELD(rdx);
10282	CHECK_FIELD(rbx);
10283	CHECK_FIELD(rsp);
10284	CHECK_FIELD(rbp);
10285	CHECK_FIELD(rsi);
10286	CHECK_FIELD(rdi);
10287	CHECK_FIELD(r8);
10288	CHECK_FIELD(r9);
10289	CHECK_FIELD(r10);
10290	CHECK_FIELD(r11);
10291	CHECK_FIELD(r12);
10292	CHECK_FIELD(r13);
10293	CHECK_SEL(cs);
10294	CHECK_SEL(ss);
10295	CHECK_SEL(ds);
10296	CHECK_SEL(es);
10297	CHECK_SEL(fs);
10298	CHECK_SEL(gs);
10299	CHECK_FIELD(cr0);
10300
10301	/* Klugde #1: REM fetches code and across the page boundrary and faults on the next page, while we execute
10302	the faulting instruction first: 001b:77f61ff3 66 8b 42 02 mov ax, word [edx+002h] (NT4SP1) */
10303	/* Kludge #2: CR2 differs slightly on cross page boundrary faults, we report the last address of the access
10304	while REM reports the address of the first byte on the page. Pending investigation as to which is correct. */
10305	if (pOrgCtx->cr2 != pDebugCtx->cr2)
10306	{
10307	if (pIemCpu->uOldCs == 0x1b && pIemCpu->uOldRip == 0x77f61ff3 && fRem)
10308	{ /* ignore */ }
10309	else if ( (pOrgCtx->cr2 & ~(uint64_t)3) == (pDebugCtx->cr2 & ~(uint64_t)3)
10310	&& (pOrgCtx->cr2 & PAGE_OFFSET_MASK) == 0
10311	&& fRem)
10312	{ /* ignore */ }
10313	else
10314	CHECK_FIELD(cr2);
10315	}
10316	CHECK_FIELD(cr3);
10317	CHECK_FIELD(cr4);
10318	CHECK_FIELD(dr[0]);
10319	CHECK_FIELD(dr[1]);
10320	CHECK_FIELD(dr[2]);
10321	CHECK_FIELD(dr[3]);
10322	CHECK_FIELD(dr[6]);
10323	if (!fRem \|\| (pOrgCtx->dr[7] & ~X86_DR7_RA1_MASK) != (pDebugCtx->dr[7] & ~X86_DR7_RA1_MASK)) /* REM 'mov drX,greg' bug.*/
10324	CHECK_FIELD(dr[7]);
10325	CHECK_FIELD(gdtr.cbGdt);
10326	CHECK_FIELD(gdtr.pGdt);
10327	CHECK_FIELD(idtr.cbIdt);
10328	CHECK_FIELD(idtr.pIdt);
10329	CHECK_SEL(ldtr);
10330	CHECK_SEL(tr);
10331	CHECK_FIELD(SysEnter.cs);
10332	CHECK_FIELD(SysEnter.eip);
10333	CHECK_FIELD(SysEnter.esp);
10334	CHECK_FIELD(msrEFER);
10335	CHECK_FIELD(msrSTAR);
10336	CHECK_FIELD(msrPAT);
10337	CHECK_FIELD(msrLSTAR);
10338	CHECK_FIELD(msrCSTAR);
10339	CHECK_FIELD(msrSFMASK);
10340	CHECK_FIELD(msrKERNELGSBASE);
10341
10342	if (cDiffs != 0)
10343	{
10344	DBGFR3Info(pVM->pUVM, "cpumguest", "verbose", NULL);
10345	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__);
10346	iemVerifyAssertMsg2(pIemCpu);
10347	RTAssertPanic();
10348	}
10349	# undef CHECK_FIELD
10350	# undef CHECK_BIT_FIELD
10351	}
10352
10353	/*
10354	* If the register state compared fine, check the verification event
10355	* records.
10356	*/
10357	if (cDiffs == 0 && !pIemCpu->fOverlappingMovs)
10358	{
10359	/*
10360	* Compare verficiation event records.
10361	* - I/O port accesses should be a 1:1 match.
10362	*/
10363	PIEMVERIFYEVTREC pIemRec = pIemCpu->pIemEvtRecHead;
10364	PIEMVERIFYEVTREC pOtherRec = pIemCpu->pOtherEvtRecHead;
10365	while (pIemRec && pOtherRec)
10366	{
10367	/* Since we might miss RAM writes and reads, ignore reads and check
10368	that any written memory is the same extra ones. */
10369	while ( IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent)
10370	&& !IEMVERIFYEVENT_IS_RAM(pOtherRec->enmEvent)
10371	&& pIemRec->pNext)
10372	{
10373	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
10374	iemVerifyWriteRecord(pIemCpu, pIemRec, fRem);
10375	pIemRec = pIemRec->pNext;
10376	}
10377
10378	/* Do the compare. */
10379	if (pIemRec->enmEvent != pOtherRec->enmEvent)
10380	{
10381	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Type mismatches");
10382	break;
10383	}
10384	bool fEquals;
10385	switch (pIemRec->enmEvent)
10386	{
10387	case IEMVERIFYEVENT_IOPORT_READ:
10388	fEquals = pIemRec->u.IOPortRead.Port == pOtherRec->u.IOPortRead.Port
10389	&& pIemRec->u.IOPortRead.cbValue == pOtherRec->u.IOPortRead.cbValue;
10390	break;
10391	case IEMVERIFYEVENT_IOPORT_WRITE:
10392	fEquals = pIemRec->u.IOPortWrite.Port == pOtherRec->u.IOPortWrite.Port
10393	&& pIemRec->u.IOPortWrite.cbValue == pOtherRec->u.IOPortWrite.cbValue
10394	&& pIemRec->u.IOPortWrite.u32Value == pOtherRec->u.IOPortWrite.u32Value;
10395	break;
10396	case IEMVERIFYEVENT_RAM_READ:
10397	fEquals = pIemRec->u.RamRead.GCPhys == pOtherRec->u.RamRead.GCPhys
10398	&& pIemRec->u.RamRead.cb == pOtherRec->u.RamRead.cb;
10399	break;
10400	case IEMVERIFYEVENT_RAM_WRITE:
10401	fEquals = pIemRec->u.RamWrite.GCPhys == pOtherRec->u.RamWrite.GCPhys
10402	&& pIemRec->u.RamWrite.cb == pOtherRec->u.RamWrite.cb
10403	&& !memcmp(pIemRec->u.RamWrite.ab, pOtherRec->u.RamWrite.ab, pIemRec->u.RamWrite.cb);
10404	break;
10405	default:
10406	fEquals = false;
10407	break;
10408	}
10409	if (!fEquals)
10410	{
10411	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Mismatch");
10412	break;
10413	}
10414
10415	/* advance */
10416	pIemRec = pIemRec->pNext;
10417	pOtherRec = pOtherRec->pNext;
10418	}
10419
10420	/* Ignore extra writes and reads. */
10421	while (pIemRec && IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent))
10422	{
10423	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
10424	iemVerifyWriteRecord(pIemCpu, pIemRec, fRem);
10425	pIemRec = pIemRec->pNext;
10426	}
10427	if (pIemRec != NULL)
10428	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra IEM record!");
10429	else if (pOtherRec != NULL)
10430	iemVerifyAssertRecord(pIemCpu, pOtherRec, "Extra Other record!");
10431	}
10432	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
10433	}
10434
10435	#else /* !IEM_VERIFICATION_MODE_FULL \|\| !IN_RING3 */
10436
10437	/* stubs */
10438	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
10439	{
10440	NOREF(pIemCpu); NOREF(Port); NOREF(pu32Value); NOREF(cbValue);
10441	return VERR_INTERNAL_ERROR;
10442	}
10443
10444	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
10445	{
10446	NOREF(pIemCpu); NOREF(Port); NOREF(u32Value); NOREF(cbValue);
10447	return VERR_INTERNAL_ERROR;
10448	}
10449
10450	#endif /* !IEM_VERIFICATION_MODE_FULL \|\| !IN_RING3 */
10451
10452
10453	#ifdef LOG_ENABLED
10454	/**
10455	* Logs the current instruction.
10456	* @param pVCpu The cross context virtual CPU structure of the caller.
10457	* @param pCtx The current CPU context.
10458	* @param fSameCtx Set if we have the same context information as the VMM,
10459	* clear if we may have already executed an instruction in
10460	* our debug context. When clear, we assume IEMCPU holds
10461	* valid CPU mode info.
10462	*/
10463	static void iemLogCurInstr(PVMCPU pVCpu, PCPUMCTX pCtx, bool fSameCtx)
10464	{
10465	# ifdef IN_RING3
10466	if (LogIs2Enabled())
10467	{
10468	char szInstr[256];
10469	uint32_t cbInstr = 0;
10470	if (fSameCtx)
10471	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
10472	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
10473	szInstr, sizeof(szInstr), &cbInstr);
10474	else
10475	{
10476	uint32_t fFlags = 0;
10477	switch (pVCpu->iem.s.enmCpuMode)
10478	{
10479	case IEMMODE_64BIT: fFlags \|= DBGF_DISAS_FLAGS_64BIT_MODE; break;
10480	case IEMMODE_32BIT: fFlags \|= DBGF_DISAS_FLAGS_32BIT_MODE; break;
10481	case IEMMODE_16BIT:
10482	if (!(pCtx->cr0 & X86_CR0_PE) \|\| pCtx->eflags.Bits.u1VM)
10483	fFlags \|= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
10484	else
10485	fFlags \|= DBGF_DISAS_FLAGS_16BIT_MODE;
10486	break;
10487	}
10488	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pCtx->cs.Sel, pCtx->rip, fFlags,
10489	szInstr, sizeof(szInstr), &cbInstr);
10490	}
10491
10492	Log2(("****\n"
10493	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
10494	" eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
10495	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
10496	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
10497	" %s\n"
10498	,
10499	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
10500	pCtx->eip, pCtx->esp, pCtx->ebp, pCtx->eflags.Bits.u2IOPL, pCtx->tr.Sel,
10501	pCtx->cs.Sel, pCtx->ss.Sel, pCtx->ds.Sel, pCtx->es.Sel,
10502	pCtx->fs.Sel, pCtx->gs.Sel, pCtx->eflags.u,
10503	pCtx->fpu.FSW, pCtx->fpu.FCW, pCtx->fpu.FTW, pCtx->fpu.MXCSR, pCtx->fpu.MXCSR_MASK,
10504	szInstr));
10505
10506	if (LogIs3Enabled())
10507	DBGFR3Info(pVCpu->pVMR3->pUVM, "cpumguest", "verbose", NULL);
10508	}
10509	else
10510	# endif
10511	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n",
10512	pCtx->cs.Sel, pCtx->rip, pCtx->ss.Sel, pCtx->rsp, pCtx->eflags.u));
10513	}
10514	#endif
10515
10516
10517	/**
10518	* Makes status code addjustments (pass up from I/O and access handler)
10519	* as well as maintaining statistics.
10520	*
10521	* @returns Strict VBox status code to pass up.
10522	* @param pIemCpu The IEM per CPU data.
10523	* @param rcStrict The status from executing an instruction.
10524	*/
10525	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PIEMCPU pIemCpu, VBOXSTRICTRC rcStrict)
10526	{
10527	if (rcStrict != VINF_SUCCESS)
10528	{
10529	if (RT_SUCCESS(rcStrict))
10530	{
10531	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
10532	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
10533	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
10534	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
10535	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
10536	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
10537	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10538	int32_t const rcPassUp = pIemCpu->rcPassUp;
10539	if (rcPassUp == VINF_SUCCESS)
10540	pIemCpu->cRetInfStatuses++;
10541	else if ( rcPassUp < VINF_EM_FIRST
10542	\|\| rcPassUp > VINF_EM_LAST
10543	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
10544	{
10545	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
10546	pIemCpu->cRetPassUpStatus++;
10547	rcStrict = rcPassUp;
10548	}
10549	else
10550	{
10551	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
10552	pIemCpu->cRetInfStatuses++;
10553	}
10554	}
10555	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
10556	pIemCpu->cRetAspectNotImplemented++;
10557	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
10558	pIemCpu->cRetInstrNotImplemented++;
10559	#ifdef IEM_VERIFICATION_MODE_FULL
10560	else if (rcStrict == VERR_IEM_RESTART_INSTRUCTION)
10561	rcStrict = VINF_SUCCESS;
10562	#endif
10563	else
10564	pIemCpu->cRetErrStatuses++;
10565	}
10566	else if (pIemCpu->rcPassUp != VINF_SUCCESS)
10567	{
10568	pIemCpu->cRetPassUpStatus++;
10569	rcStrict = pIemCpu->rcPassUp;
10570	}
10571
10572	return rcStrict;
10573	}
10574
10575
10576	/**
10577	* The actual code execution bits of IEMExecOne, IEMExecOneEx, and
10578	* IEMExecOneWithPrefetchedByPC.
10579	*
10580	* @return Strict VBox status code.
10581	* @param pVCpu The current virtual CPU.
10582	* @param pIemCpu The IEM per CPU data.
10583	* @param fExecuteInhibit If set, execute the instruction following CLI,
10584	* POP SS and MOV SS,GR.
10585	*/
10586	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPU pVCpu, PIEMCPU pIemCpu, bool fExecuteInhibit)
10587	{
10588	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10589	VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10590	if (rcStrict == VINF_SUCCESS)
10591	pIemCpu->cInstructions++;
10592	if (pIemCpu->cActiveMappings > 0)
10593	iemMemRollback(pIemCpu);
10594	//#ifdef DEBUG
10595	// AssertMsg(pIemCpu->offOpcode == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", pIemCpu->offOpcode, cbInstr));
10596	//#endif
10597
10598	/* Execute the next instruction as well if a cli, pop ss or
10599	mov ss, Gr has just completed successfully. */
10600	if ( fExecuteInhibit
10601	&& rcStrict == VINF_SUCCESS
10602	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10603	&& EMGetInhibitInterruptsPC(pVCpu) == pIemCpu->CTX_SUFF(pCtx)->rip )
10604	{
10605	rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, pIemCpu->fBypassHandlers);
10606	if (rcStrict == VINF_SUCCESS)
10607	{
10608	# ifdef LOG_ENABLED
10609	iemLogCurInstr(IEMCPU_TO_VMCPU(pIemCpu), pIemCpu->CTX_SUFF(pCtx), false);
10610	# endif
10611	IEM_OPCODE_GET_NEXT_U8(&b);
10612	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10613	if (rcStrict == VINF_SUCCESS)
10614	pIemCpu->cInstructions++;
10615	if (pIemCpu->cActiveMappings > 0)
10616	iemMemRollback(pIemCpu);
10617	}
10618	EMSetInhibitInterruptsPC(pVCpu, UINT64_C(0x7777555533331111));
10619	}
10620
10621	/*
10622	* Return value fiddling, statistics and sanity assertions.
10623	*/
10624	rcStrict = iemExecStatusCodeFiddling(pIemCpu, rcStrict);
10625
10626	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pIemCpu->CTX_SUFF(pCtx)->cs));
10627	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pIemCpu->CTX_SUFF(pCtx)->ss));
10628	#if defined(IEM_VERIFICATION_MODE_FULL)
10629	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pIemCpu->CTX_SUFF(pCtx)->es));
10630	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pIemCpu->CTX_SUFF(pCtx)->ds));
10631	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pIemCpu->CTX_SUFF(pCtx)->fs));
10632	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pIemCpu->CTX_SUFF(pCtx)->gs));
10633	#endif
10634	return rcStrict;
10635	}
10636
10637
10638	#ifdef IN_RC
10639	/**
10640	* Re-enters raw-mode or ensure we return to ring-3.
10641	*
10642	* @returns rcStrict, maybe modified.
10643	* @param pIemCpu The IEM CPU structure.
10644	* @param pVCpu The cross context virtual CPU structure of the caller.
10645	* @param pCtx The current CPU context.
10646	* @param rcStrict The status code returne by the interpreter.
10647	*/
10648	DECLINLINE(VBOXSTRICTRC) iemRCRawMaybeReenter(PIEMCPU pIemCpu, PVMCPU pVCpu, PCPUMCTX pCtx, VBOXSTRICTRC rcStrict)
10649	{
10650	if (!pIemCpu->fInPatchCode)
10651	CPUMRawEnter(pVCpu, CPUMCTX2CORE(pCtx));
10652	return rcStrict;
10653	}
10654	#endif
10655
10656
10657	/**
10658	* Execute one instruction.
10659	*
10660	* @return Strict VBox status code.
10661	* @param pVCpu The current virtual CPU.
10662	*/
10663	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPU pVCpu)
10664	{
10665	PIEMCPU pIemCpu = &pVCpu->iem.s;
10666
10667	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
10668	iemExecVerificationModeSetup(pIemCpu);
10669	#endif
10670	#ifdef LOG_ENABLED
10671	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
10672	iemLogCurInstr(pVCpu, pCtx, true);
10673	#endif
10674
10675	/*
10676	* Do the decoding and emulation.
10677	*/
10678	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, false);
10679	if (rcStrict == VINF_SUCCESS)
10680	rcStrict = iemExecOneInner(pVCpu, pIemCpu, true);
10681
10682	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
10683	/*
10684	* Assert some sanity.
10685	*/
10686	iemExecVerificationModeCheck(pIemCpu);
10687	#endif
10688	#ifdef IN_RC
10689	rcStrict = iemRCRawMaybeReenter(pIemCpu, pVCpu, pIemCpu->CTX_SUFF(pCtx), rcStrict);
10690	#endif
10691	if (rcStrict != VINF_SUCCESS)
10692	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10693	pCtx->cs.Sel, pCtx->rip, pCtx->ss.Sel, pCtx->rsp, pCtx->eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10694	return rcStrict;
10695	}
10696
10697
10698	VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
10699	{
10700	PIEMCPU pIemCpu = &pVCpu->iem.s;
10701	PCPUMCTX pCtx = pVCpu->iem.s.CTX_SUFF(pCtx);
10702	AssertReturn(CPUMCTX2CORE(pCtx) == pCtxCore, VERR_IEM_IPE_3);
10703
10704	uint32_t const cbOldWritten = pIemCpu->cbWritten;
10705	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, false);
10706	if (rcStrict == VINF_SUCCESS)
10707	{
10708	rcStrict = iemExecOneInner(pVCpu, pIemCpu, true);
10709	if (pcbWritten)
10710	*pcbWritten = pIemCpu->cbWritten - cbOldWritten;
10711	}
10712
10713	#ifdef IN_RC
10714	rcStrict = iemRCRawMaybeReenter(pIemCpu, pVCpu, pCtx, rcStrict);
10715	#endif
10716	return rcStrict;
10717	}
10718
10719
10720	VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
10721	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
10722	{
10723	PIEMCPU pIemCpu = &pVCpu->iem.s;
10724	PCPUMCTX pCtx = pVCpu->iem.s.CTX_SUFF(pCtx);
10725	AssertReturn(CPUMCTX2CORE(pCtx) == pCtxCore, VERR_IEM_IPE_3);
10726
10727	VBOXSTRICTRC rcStrict;
10728	if ( cbOpcodeBytes
10729	&& pCtx->rip == OpcodeBytesPC)
10730	{
10731	iemInitDecoder(pIemCpu, false);
10732	pIemCpu->cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pIemCpu->abOpcode));
10733	memcpy(pIemCpu->abOpcode, pvOpcodeBytes, pIemCpu->cbOpcode);
10734	rcStrict = VINF_SUCCESS;
10735	}
10736	else
10737	rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, false);
10738	if (rcStrict == VINF_SUCCESS)
10739	{
10740	rcStrict = iemExecOneInner(pVCpu, pIemCpu, true);
10741	}
10742
10743	#ifdef IN_RC
10744	rcStrict = iemRCRawMaybeReenter(pIemCpu, pVCpu, pCtx, rcStrict);
10745	#endif
10746	return rcStrict;
10747	}
10748
10749
10750	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
10751	{
10752	PIEMCPU pIemCpu = &pVCpu->iem.s;
10753	PCPUMCTX pCtx = pVCpu->iem.s.CTX_SUFF(pCtx);
10754	AssertReturn(CPUMCTX2CORE(pCtx) == pCtxCore, VERR_IEM_IPE_3);
10755
10756	uint32_t const cbOldWritten = pIemCpu->cbWritten;
10757	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, true);
10758	if (rcStrict == VINF_SUCCESS)
10759	{
10760	rcStrict = iemExecOneInner(pVCpu, pIemCpu, false);
10761	if (pcbWritten)
10762	*pcbWritten = pIemCpu->cbWritten - cbOldWritten;
10763	}
10764
10765	#ifdef IN_RC
10766	rcStrict = iemRCRawMaybeReenter(pIemCpu, pVCpu, pCtx, rcStrict);
10767	#endif
10768	return rcStrict;
10769	}
10770
10771
10772	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
10773	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
10774	{
10775	PIEMCPU pIemCpu = &pVCpu->iem.s;
10776	PCPUMCTX pCtx = pVCpu->iem.s.CTX_SUFF(pCtx);
10777	AssertReturn(CPUMCTX2CORE(pCtx) == pCtxCore, VERR_IEM_IPE_3);
10778
10779	VBOXSTRICTRC rcStrict;
10780	if ( cbOpcodeBytes
10781	&& pCtx->rip == OpcodeBytesPC)
10782	{
10783	iemInitDecoder(pIemCpu, true);
10784	pIemCpu->cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pIemCpu->abOpcode));
10785	memcpy(pIemCpu->abOpcode, pvOpcodeBytes, pIemCpu->cbOpcode);
10786	rcStrict = VINF_SUCCESS;
10787	}
10788	else
10789	rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, true);
10790	if (rcStrict == VINF_SUCCESS)
10791	rcStrict = iemExecOneInner(pVCpu, pIemCpu, false);
10792
10793	#ifdef IN_RC
10794	rcStrict = iemRCRawMaybeReenter(pIemCpu, pVCpu, pCtx, rcStrict);
10795	#endif
10796	return rcStrict;
10797	}
10798
10799
10800	VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPU pVCpu)
10801	{
10802	PIEMCPU pIemCpu = &pVCpu->iem.s;
10803
10804	/*
10805	* See if there is an interrupt pending in TRPM and inject it if we can.
10806	*/
10807	#if !defined(IEM_VERIFICATION_MODE_FULL) \|\| !defined(IN_RING3)
10808	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
10809	# ifdef IEM_VERIFICATION_MODE_FULL
10810	pIemCpu->uInjectCpl = UINT8_MAX;
10811	# endif
10812	if ( pCtx->eflags.Bits.u1IF
10813	&& TRPMHasTrap(pVCpu)
10814	&& EMGetInhibitInterruptsPC(pVCpu) != pCtx->rip)
10815	{
10816	uint8_t u8TrapNo;
10817	TRPMEVENT enmType;
10818	RTGCUINT uErrCode;
10819	RTGCPTR uCr2;
10820	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */); AssertRC(rc2);
10821	IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
10822	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
10823	TRPMResetTrap(pVCpu);
10824	}
10825	#else
10826	iemExecVerificationModeSetup(pIemCpu);
10827	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
10828	#endif
10829
10830	/*
10831	* Log the state.
10832	*/
10833	#ifdef LOG_ENABLED
10834	iemLogCurInstr(pVCpu, pCtx, true);
10835	#endif
10836
10837	/*
10838	* Do the decoding and emulation.
10839	*/
10840	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu, false);
10841	if (rcStrict == VINF_SUCCESS)
10842	rcStrict = iemExecOneInner(pVCpu, pIemCpu, true);
10843
10844	#if defined(IEM_VERIFICATION_MODE_FULL) && defined(IN_RING3)
10845	/*
10846	* Assert some sanity.
10847	*/
10848	iemExecVerificationModeCheck(pIemCpu);
10849	#endif
10850
10851	/*
10852	* Maybe re-enter raw-mode and log.
10853	*/
10854	#ifdef IN_RC
10855	rcStrict = iemRCRawMaybeReenter(pIemCpu, pVCpu, pIemCpu->CTX_SUFF(pCtx), rcStrict);
10856	#endif
10857	if (rcStrict != VINF_SUCCESS)
10858	LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10859	pCtx->cs.Sel, pCtx->rip, pCtx->ss.Sel, pCtx->rsp, pCtx->eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10860	return rcStrict;
10861	}
10862
10863
10864
10865	/**
10866	* Injects a trap, fault, abort, software interrupt or external interrupt.
10867	*
10868	* The parameter list matches TRPMQueryTrapAll pretty closely.
10869	*
10870	* @returns Strict VBox status code.
10871	* @param pVCpu The current virtual CPU.
10872	* @param u8TrapNo The trap number.
10873	* @param enmType What type is it (trap/fault/abort), software
10874	* interrupt or hardware interrupt.
10875	* @param uErrCode The error code if applicable.
10876	* @param uCr2 The CR2 value if applicable.
10877	* @param cbInstr The instruction length (only relevant for
10878	* software interrupts).
10879	*/
10880	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPU pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
10881	uint8_t cbInstr)
10882	{
10883	iemInitDecoder(&pVCpu->iem.s, false);
10884	#ifdef DBGFTRACE_ENABLED
10885	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
10886	u8TrapNo, enmType, uErrCode, uCr2);
10887	#endif
10888
10889	uint32_t fFlags;
10890	switch (enmType)
10891	{
10892	case TRPM_HARDWARE_INT:
10893	Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
10894	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
10895	uErrCode = uCr2 = 0;
10896	break;
10897
10898	case TRPM_SOFTWARE_INT:
10899	Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
10900	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
10901	uErrCode = uCr2 = 0;
10902	break;
10903
10904	case TRPM_TRAP:
10905	Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
10906	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
10907	if (u8TrapNo == X86_XCPT_PF)
10908	fFlags \|= IEM_XCPT_FLAGS_CR2;
10909	switch (u8TrapNo)
10910	{
10911	case X86_XCPT_DF:
10912	case X86_XCPT_TS:
10913	case X86_XCPT_NP:
10914	case X86_XCPT_SS:
10915	case X86_XCPT_PF:
10916	case X86_XCPT_AC:
10917	fFlags \|= IEM_XCPT_FLAGS_ERR;
10918	break;
10919
10920	case X86_XCPT_NMI:
10921	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
10922	break;
10923	}
10924	break;
10925
10926	IEM_NOT_REACHED_DEFAULT_CASE_RET();
10927	}
10928
10929	return iemRaiseXcptOrInt(&pVCpu->iem.s, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
10930	}
10931
10932
10933	/**
10934	* Injects the active TRPM event.
10935	*
10936	* @returns Strict VBox status code.
10937	* @param pVCpu Pointer to the VMCPU.
10938	*/
10939	VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPU pVCpu)
10940	{
10941	#ifndef IEM_IMPLEMENTS_TASKSWITCH
10942	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
10943	#else
10944	uint8_t u8TrapNo;
10945	TRPMEVENT enmType;
10946	RTGCUINT uErrCode;
10947	RTGCUINTPTR uCr2;
10948	uint8_t cbInstr;
10949	int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr);
10950	if (RT_FAILURE(rc))
10951	return rc;
10952
10953	TRPMResetTrap(pVCpu);
10954	return IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
10955	#endif
10956	}
10957
10958
10959	VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
10960	{
10961	return VERR_NOT_IMPLEMENTED;
10962	}
10963
10964
10965	VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
10966	{
10967	return VERR_NOT_IMPLEMENTED;
10968	}
10969
10970
10971	#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
10972	/**
10973	* Executes a IRET instruction with default operand size.
10974	*
10975	* This is for PATM.
10976	*
10977	* @returns VBox status code.
10978	* @param pVCpu The current virtual CPU.
10979	* @param pCtxCore The register frame.
10980	*/
10981	VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore)
10982	{
10983	PIEMCPU pIemCpu = &pVCpu->iem.s;
10984	PCPUMCTX pCtx = pVCpu->iem.s.CTX_SUFF(pCtx);
10985
10986	iemCtxCoreToCtx(pCtx, pCtxCore);
10987	iemInitDecoder(pIemCpu);
10988	VBOXSTRICTRC rcStrict = iemCImpl_iret(pIemCpu, 1, pIemCpu->enmDefOpSize);
10989	if (rcStrict == VINF_SUCCESS)
10990	iemCtxToCtxCore(pCtxCore, pCtx);
10991	else
10992	LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10993	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10994	return rcStrict;
10995	}
10996	#endif
10997
10998
10999
11000	/**
11001	* Interface for HM and EM for executing string I/O OUT (write) instructions.
11002	*
11003	* This API ASSUMES that the caller has already verified that the guest code is
11004	* allowed to access the I/O port. (The I/O port is in the DX register in the
11005	* guest state.)
11006	*
11007	* @returns Strict VBox status code.
11008	* @param pVCpu The cross context per virtual CPU structure.
11009	* @param cbValue The size of the I/O port access (1, 2, or 4).
11010	* @param enmAddrMode The addressing mode.
11011	* @param fRepPrefix Indicates whether a repeat prefix is used
11012	* (doesn't matter which for this instruction).
11013	* @param cbInstr The instruction length in bytes.
11014	* @param iEffSeg The effective segment address.
11015	*/
11016	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPU pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
11017	bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg)
11018	{
11019	AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
11020	AssertReturn(cbInstr - 1U <= 14U, VERR_IEM_INVALID_INSTR_LENGTH);
11021
11022	/*
11023	* State init.
11024	*/
11025	PIEMCPU pIemCpu = &pVCpu->iem.s;
11026	iemInitExec(pIemCpu, false /fBypassHandlers/);
11027
11028	/*
11029	* Switch orgy for getting to the right handler.
11030	*/
11031	VBOXSTRICTRC rcStrict;
11032	if (fRepPrefix)
11033	{
11034	switch (enmAddrMode)
11035	{
11036	case IEMMODE_16BIT:
11037	switch (cbValue)
11038	{
11039	case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11040	case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11041	case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11042	default:
11043	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11044	}
11045	break;
11046
11047	case IEMMODE_32BIT:
11048	switch (cbValue)
11049	{
11050	case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11051	case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11052	case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11053	default:
11054	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11055	}
11056	break;
11057
11058	case IEMMODE_64BIT:
11059	switch (cbValue)
11060	{
11061	case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11062	case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11063	case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11064	default:
11065	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11066	}
11067	break;
11068
11069	default:
11070	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
11071	}
11072	}
11073	else
11074	{
11075	switch (enmAddrMode)
11076	{
11077	case IEMMODE_16BIT:
11078	switch (cbValue)
11079	{
11080	case 1: rcStrict = iemCImpl_outs_op8_addr16(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11081	case 2: rcStrict = iemCImpl_outs_op16_addr16(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11082	case 4: rcStrict = iemCImpl_outs_op32_addr16(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11083	default:
11084	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11085	}
11086	break;
11087
11088	case IEMMODE_32BIT:
11089	switch (cbValue)
11090	{
11091	case 1: rcStrict = iemCImpl_outs_op8_addr32(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11092	case 2: rcStrict = iemCImpl_outs_op16_addr32(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11093	case 4: rcStrict = iemCImpl_outs_op32_addr32(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11094	default:
11095	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11096	}
11097	break;
11098
11099	case IEMMODE_64BIT:
11100	switch (cbValue)
11101	{
11102	case 1: rcStrict = iemCImpl_outs_op8_addr64(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11103	case 2: rcStrict = iemCImpl_outs_op16_addr64(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11104	case 4: rcStrict = iemCImpl_outs_op32_addr64(pIemCpu, cbInstr, iEffSeg, true /fIoChecked/); break;
11105	default:
11106	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11107	}
11108	break;
11109
11110	default:
11111	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
11112	}
11113	}
11114
11115	return iemExecStatusCodeFiddling(pIemCpu, rcStrict);
11116	}
11117
11118
11119	/**
11120	* Interface for HM and EM for executing string I/O IN (read) instructions.
11121	*
11122	* This API ASSUMES that the caller has already verified that the guest code is
11123	* allowed to access the I/O port. (The I/O port is in the DX register in the
11124	* guest state.)
11125	*
11126	* @returns Strict VBox status code.
11127	* @param pVCpu The cross context per virtual CPU structure.
11128	* @param cbValue The size of the I/O port access (1, 2, or 4).
11129	* @param enmAddrMode The addressing mode.
11130	* @param fRepPrefix Indicates whether a repeat prefix is used
11131	* (doesn't matter which for this instruction).
11132	* @param cbInstr The instruction length in bytes.
11133	*/
11134	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPU pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
11135	bool fRepPrefix, uint8_t cbInstr)
11136	{
11137	AssertReturn(cbInstr - 1U <= 14U, VERR_IEM_INVALID_INSTR_LENGTH);
11138
11139	/*
11140	* State init.
11141	*/
11142	PIEMCPU pIemCpu = &pVCpu->iem.s;
11143	iemInitExec(pIemCpu, false /fBypassHandlers/);
11144
11145	/*
11146	* Switch orgy for getting to the right handler.
11147	*/
11148	VBOXSTRICTRC rcStrict;
11149	if (fRepPrefix)
11150	{
11151	switch (enmAddrMode)
11152	{
11153	case IEMMODE_16BIT:
11154	switch (cbValue)
11155	{
11156	case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pIemCpu, cbInstr, true /fIoChecked/); break;
11157	case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pIemCpu, cbInstr, true /fIoChecked/); break;
11158	case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pIemCpu, cbInstr, true /fIoChecked/); break;
11159	default:
11160	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11161	}
11162	break;
11163
11164	case IEMMODE_32BIT:
11165	switch (cbValue)
11166	{
11167	case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pIemCpu, cbInstr, true /fIoChecked/); break;
11168	case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pIemCpu, cbInstr, true /fIoChecked/); break;
11169	case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pIemCpu, cbInstr, true /fIoChecked/); break;
11170	default:
11171	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11172	}
11173	break;
11174
11175	case IEMMODE_64BIT:
11176	switch (cbValue)
11177	{
11178	case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pIemCpu, cbInstr, true /fIoChecked/); break;
11179	case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pIemCpu, cbInstr, true /fIoChecked/); break;
11180	case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pIemCpu, cbInstr, true /fIoChecked/); break;
11181	default:
11182	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11183	}
11184	break;
11185
11186	default:
11187	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
11188	}
11189	}
11190	else
11191	{
11192	switch (enmAddrMode)
11193	{
11194	case IEMMODE_16BIT:
11195	switch (cbValue)
11196	{
11197	case 1: rcStrict = iemCImpl_ins_op8_addr16(pIemCpu, cbInstr, true /fIoChecked/); break;
11198	case 2: rcStrict = iemCImpl_ins_op16_addr16(pIemCpu, cbInstr, true /fIoChecked/); break;
11199	case 4: rcStrict = iemCImpl_ins_op32_addr16(pIemCpu, cbInstr, true /fIoChecked/); break;
11200	default:
11201	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11202	}
11203	break;
11204
11205	case IEMMODE_32BIT:
11206	switch (cbValue)
11207	{
11208	case 1: rcStrict = iemCImpl_ins_op8_addr32(pIemCpu, cbInstr, true /fIoChecked/); break;
11209	case 2: rcStrict = iemCImpl_ins_op16_addr32(pIemCpu, cbInstr, true /fIoChecked/); break;
11210	case 4: rcStrict = iemCImpl_ins_op32_addr32(pIemCpu, cbInstr, true /fIoChecked/); break;
11211	default:
11212	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11213	}
11214	break;
11215
11216	case IEMMODE_64BIT:
11217	switch (cbValue)
11218	{
11219	case 1: rcStrict = iemCImpl_ins_op8_addr64(pIemCpu, cbInstr, true /fIoChecked/); break;
11220	case 2: rcStrict = iemCImpl_ins_op16_addr64(pIemCpu, cbInstr, true /fIoChecked/); break;
11221	case 4: rcStrict = iemCImpl_ins_op32_addr64(pIemCpu, cbInstr, true /fIoChecked/); break;
11222	default:
11223	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
11224	}
11225	break;
11226
11227	default:
11228	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
11229	}
11230	}
11231
11232	return iemExecStatusCodeFiddling(pIemCpu, rcStrict);
11233	}
11234

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 52077

Download in other formats: