IEMAll.cpp@ 40256

Last change on this file since 40256 was 40256, checked in by vboxsync, 13 years ago
IEM: fnstsw m16, ffree and ffreep, reimplemented fincstp and fdecstp.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 285.1 KB

Line
1	/* $Id: IEMAll.cpp 40256 2012-02-25 01:09:31Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/** @page pg_iem IEM - Interpreted Execution Manager
20	*
21	* The interpreted exeuction manager (IEM) is for executing short guest code
22	* sequences that are causing too many exits / virtualization traps. It will
23	* also be used to interpret single instructions, thus replacing the selective
24	* interpreters in EM and IOM.
25	*
26	* Design goals:
27	* - Relatively small footprint, although we favour speed and correctness
28	* over size.
29	* - Reasonably fast.
30	* - Correctly handle lock prefixed instructions.
31	* - Complete instruction set - eventually.
32	* - Refactorable into a recompiler, maybe.
33	* - Replace EMInterpret*.
34	*
35	* Using the existing disassembler has been considered, however this is thought
36	* to conflict with speed as the disassembler chews things a bit too much while
37	* leaving us with a somewhat complicated state to interpret afterwards.
38	*
39	*
40	* The current code is very much work in progress. You've been warned!
41	*
42	*
43	* @section sec_iem_fpu_instr FPU Instructions
44	*
45	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46	* same or equivalent instructions on the host FPU. To make life easy, we also
47	* let the FPU prioritize the unmasked exceptions for us. This however, only
48	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
50	* can trigger spurious FPU exceptions.
51	*
52	* The guest FPU state is not loaded into the host CPU and kept there till we
53	* leave IEM because the calling conventions have declared an all year open
54	* season on much of the FPU state. For instance an innocent looking call to
55	* memcpy might end up using a whole bunch of XMM or MM registers if the
56	* particular implementation finds it worthwhile.
57	*
58	*
59	* @section sec_iem_logging Logging
60	*
61	* The IEM code uses the \"IEM\" log group for the main logging. The different
62	* logging levels/flags are generally used for the following purposes:
63	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64	* - Flow (LogFlow): Additional exception details, basic enter/exit IEM
65	* state info.
66	* - Level 2 (Log2): ?
67	* - Level 3 (Log3): More detailed enter/exit IEM state info.
68	* - Level 4 (Log4): Decoding mnemonics w/ EIP.
69	* - Level 5 (Log5): Decoding details.
70	* - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
71	*
72	*/
73
74	/*******************************************************************************
75	* Header Files *
76	*******************************************************************************/
77	#define LOG_GROUP LOG_GROUP_IEM
78	#include <VBox/vmm/iem.h>
79	#include <VBox/vmm/pgm.h>
80	#include <VBox/vmm/iom.h>
81	#include <VBox/vmm/em.h>
82	#include <VBox/vmm/tm.h>
83	#include <VBox/vmm/dbgf.h>
84	#ifdef IEM_VERIFICATION_MODE
85	# include <VBox/vmm/rem.h>
86	# include <VBox/vmm/mm.h>
87	#endif
88	#include "IEMInternal.h"
89	#include <VBox/vmm/vm.h>
90	#include <VBox/log.h>
91	#include <VBox/err.h>
92	#include <VBox/param.h>
93	#include <iprt/assert.h>
94	#include <iprt/string.h>
95	#include <iprt/x86.h>
96
97
98	/*******************************************************************************
99	* Structures and Typedefs *
100	*******************************************************************************/
101	/** @typedef PFNIEMOP
102	* Pointer to an opcode decoder function.
103	*/
104
105	/** @def FNIEMOP_DEF
106	* Define an opcode decoder function.
107	*
108	* We're using macors for this so that adding and removing parameters as well as
109	* tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
110	*
111	* @param a_Name The function name.
112	*/
113
114
115	#if defined(__GNUC__) && defined(RT_ARCH_X86)
116	typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PIEMCPU pIemCpu);
117	# define FNIEMOP_DEF(a_Name) \
118	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name (PIEMCPU pIemCpu)
119	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
120	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
121	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
122	static VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
123
124	#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
125	typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PIEMCPU pIemCpu);
126	# define FNIEMOP_DEF(a_Name) \
127	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu) RT_NO_THROW
128	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
129	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
130	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
131	static /__declspec(naked)/ VBOXSTRICTRC __fastcall a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
132
133	#elif defined(__GNUC__)
134	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
135	# define FNIEMOP_DEF(a_Name) \
136	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu)
137	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
138	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0)
139	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
140	static VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1)
141
142	#else
143	typedef VBOXSTRICTRC (* PFNIEMOP)(PIEMCPU pIemCpu);
144	# define FNIEMOP_DEF(a_Name) \
145	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu) RT_NO_THROW
146	# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
147	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0) RT_NO_THROW
148	# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
149	static VBOXSTRICTRC a_Name(PIEMCPU pIemCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW
150
151	#endif
152
153
154	/**
155	* Selector descriptor table entry as fetched by iemMemFetchSelDesc.
156	*/
157	typedef union IEMSELDESC
158	{
159	/** The legacy view. */
160	X86DESC Legacy;
161	/** The long mode view. */
162	X86DESC64 Long;
163	} IEMSELDESC;
164	/** Pointer to a selector descriptor table entry. */
165	typedef IEMSELDESC *PIEMSELDESC;
166
167
168	/*******************************************************************************
169	* Defined Constants And Macros *
170	*******************************************************************************/
171	/** @name IEM status codes.
172	*
173	* Not quite sure how this will play out in the end, just aliasing safe status
174	* codes for now.
175	*
176	* @{ */
177	#define VINF_IEM_RAISED_XCPT VINF_EM_RESCHEDULE
178	/** @} */
179
180	/** Temporary hack to disable the double execution. Will be removed in favor
181	* of a dedicated execution mode in EM. */
182	//#define IEM_VERIFICATION_MODE_NO_REM
183
184	/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
185	* due to GCC lacking knowledge about the value range of a switch. */
186	#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
187
188	/**
189	* Call an opcode decoder function.
190	*
191	* We're using macors for this so that adding and removing parameters can be
192	* done as we please. See FNIEMOP_DEF.
193	*/
194	#define FNIEMOP_CALL(a_pfn) (a_pfn)(pIemCpu)
195
196	/**
197	* Call a common opcode decoder function taking one extra argument.
198	*
199	* We're using macors for this so that adding and removing parameters can be
200	* done as we please. See FNIEMOP_DEF_1.
201	*/
202	#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pIemCpu, a0)
203
204	/**
205	* Call a common opcode decoder function taking one extra argument.
206	*
207	* We're using macors for this so that adding and removing parameters can be
208	* done as we please. See FNIEMOP_DEF_1.
209	*/
210	#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pIemCpu, a0, a1)
211
212	/**
213	* Check if we're currently executing in real or virtual 8086 mode.
214	*
215	* @returns @c true if it is, @c false if not.
216	* @param a_pIemCpu The IEM state of the current CPU.
217	*/
218	#define IEM_IS_REAL_OR_V86_MODE(a_pIemCpu) (CPUMIsGuestInRealOrV86ModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
219
220	/**
221	* Check if we're currently executing in long mode.
222	*
223	* @returns @c true if it is, @c false if not.
224	* @param a_pIemCpu The IEM state of the current CPU.
225	*/
226	#define IEM_IS_LONG_MODE(a_pIemCpu) (CPUMIsGuestInLongModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
227
228	/**
229	* Check if we're currently executing in real mode.
230	*
231	* @returns @c true if it is, @c false if not.
232	* @param a_pIemCpu The IEM state of the current CPU.
233	*/
234	#define IEM_IS_REAL_MODE(a_pIemCpu) (CPUMIsGuestInRealModeEx((a_pIemCpu)->CTX_SUFF(pCtx)))
235
236	/**
237	* Tests if an AMD CPUID feature (extended) is marked present - ECX.
238	*/
239	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX(a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, 0, (a_fEcx))
240
241	/**
242	* Tests if an AMD CPUID feature (extended) is marked present - EDX.
243	*/
244	#define IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX(a_fEdx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0)
245
246	/**
247	* Tests if at least on of the specified AMD CPUID features (extended) are
248	* marked present.
249	*/
250	#define IEM_IS_AMD_CPUID_FEATURES_ANY_PRESENT(a_fEdx, a_fEcx) iemRegIsAmdCpuIdFeaturePresent(pIemCpu, (a_fEdx), (a_fEcx))
251
252	/**
253	* Checks if a intel CPUID feature is present.
254	*/
255	#define IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX(a_fEdx) \
256	( ((a_fEdx) & (X86_CPUID_FEATURE_EDX_TSC \| 0)) \
257	\|\| iemRegIsIntelCpuIdFeaturePresent(pIemCpu, (a_fEdx), 0) )
258
259	/**
260	* Check if the address is canonical.
261	*/
262	#define IEM_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
263
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
269
270
271	/** Function table for the ADD instruction. */
272	static const IEMOPBINSIZES g_iemAImpl_add =
273	{
274	iemAImpl_add_u8, iemAImpl_add_u8_locked,
275	iemAImpl_add_u16, iemAImpl_add_u16_locked,
276	iemAImpl_add_u32, iemAImpl_add_u32_locked,
277	iemAImpl_add_u64, iemAImpl_add_u64_locked
278	};
279
280	/** Function table for the ADC instruction. */
281	static const IEMOPBINSIZES g_iemAImpl_adc =
282	{
283	iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
284	iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
285	iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
286	iemAImpl_adc_u64, iemAImpl_adc_u64_locked
287	};
288
289	/** Function table for the SUB instruction. */
290	static const IEMOPBINSIZES g_iemAImpl_sub =
291	{
292	iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
293	iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
294	iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
295	iemAImpl_sub_u64, iemAImpl_sub_u64_locked
296	};
297
298	/** Function table for the SBB instruction. */
299	static const IEMOPBINSIZES g_iemAImpl_sbb =
300	{
301	iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
302	iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
303	iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
304	iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
305	};
306
307	/** Function table for the OR instruction. */
308	static const IEMOPBINSIZES g_iemAImpl_or =
309	{
310	iemAImpl_or_u8, iemAImpl_or_u8_locked,
311	iemAImpl_or_u16, iemAImpl_or_u16_locked,
312	iemAImpl_or_u32, iemAImpl_or_u32_locked,
313	iemAImpl_or_u64, iemAImpl_or_u64_locked
314	};
315
316	/** Function table for the XOR instruction. */
317	static const IEMOPBINSIZES g_iemAImpl_xor =
318	{
319	iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
320	iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
321	iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
322	iemAImpl_xor_u64, iemAImpl_xor_u64_locked
323	};
324
325	/** Function table for the AND instruction. */
326	static const IEMOPBINSIZES g_iemAImpl_and =
327	{
328	iemAImpl_and_u8, iemAImpl_and_u8_locked,
329	iemAImpl_and_u16, iemAImpl_and_u16_locked,
330	iemAImpl_and_u32, iemAImpl_and_u32_locked,
331	iemAImpl_and_u64, iemAImpl_and_u64_locked
332	};
333
334	/** Function table for the CMP instruction.
335	* @remarks Making operand order ASSUMPTIONS.
336	*/
337	static const IEMOPBINSIZES g_iemAImpl_cmp =
338	{
339	iemAImpl_cmp_u8, NULL,
340	iemAImpl_cmp_u16, NULL,
341	iemAImpl_cmp_u32, NULL,
342	iemAImpl_cmp_u64, NULL
343	};
344
345	/** Function table for the TEST instruction.
346	* @remarks Making operand order ASSUMPTIONS.
347	*/
348	static const IEMOPBINSIZES g_iemAImpl_test =
349	{
350	iemAImpl_test_u8, NULL,
351	iemAImpl_test_u16, NULL,
352	iemAImpl_test_u32, NULL,
353	iemAImpl_test_u64, NULL
354	};
355
356	/** Function table for the BT instruction. */
357	static const IEMOPBINSIZES g_iemAImpl_bt =
358	{
359	NULL, NULL,
360	iemAImpl_bt_u16, NULL,
361	iemAImpl_bt_u32, NULL,
362	iemAImpl_bt_u64, NULL
363	};
364
365	/** Function table for the BTC instruction. */
366	static const IEMOPBINSIZES g_iemAImpl_btc =
367	{
368	NULL, NULL,
369	iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
370	iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
371	iemAImpl_btc_u64, iemAImpl_btc_u64_locked
372	};
373
374	/** Function table for the BTR instruction. */
375	static const IEMOPBINSIZES g_iemAImpl_btr =
376	{
377	NULL, NULL,
378	iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
379	iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
380	iemAImpl_btr_u64, iemAImpl_btr_u64_locked
381	};
382
383	/** Function table for the BTS instruction. */
384	static const IEMOPBINSIZES g_iemAImpl_bts =
385	{
386	NULL, NULL,
387	iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
388	iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
389	iemAImpl_bts_u64, iemAImpl_bts_u64_locked
390	};
391
392	/** Function table for the BSF instruction. */
393	static const IEMOPBINSIZES g_iemAImpl_bsf =
394	{
395	NULL, NULL,
396	iemAImpl_bsf_u16, NULL,
397	iemAImpl_bsf_u32, NULL,
398	iemAImpl_bsf_u64, NULL
399	};
400
401	/** Function table for the BSR instruction. */
402	static const IEMOPBINSIZES g_iemAImpl_bsr =
403	{
404	NULL, NULL,
405	iemAImpl_bsr_u16, NULL,
406	iemAImpl_bsr_u32, NULL,
407	iemAImpl_bsr_u64, NULL
408	};
409
410	/** Function table for the IMUL instruction. */
411	static const IEMOPBINSIZES g_iemAImpl_imul_two =
412	{
413	NULL, NULL,
414	iemAImpl_imul_two_u16, NULL,
415	iemAImpl_imul_two_u32, NULL,
416	iemAImpl_imul_two_u64, NULL
417	};
418
419	/** Group 1 /r lookup table. */
420	static const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
421	{
422	&g_iemAImpl_add,
423	&g_iemAImpl_or,
424	&g_iemAImpl_adc,
425	&g_iemAImpl_sbb,
426	&g_iemAImpl_and,
427	&g_iemAImpl_sub,
428	&g_iemAImpl_xor,
429	&g_iemAImpl_cmp
430	};
431
432	/** Function table for the INC instruction. */
433	static const IEMOPUNARYSIZES g_iemAImpl_inc =
434	{
435	iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
436	iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
437	iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
438	iemAImpl_inc_u64, iemAImpl_inc_u64_locked
439	};
440
441	/** Function table for the DEC instruction. */
442	static const IEMOPUNARYSIZES g_iemAImpl_dec =
443	{
444	iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
445	iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
446	iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
447	iemAImpl_dec_u64, iemAImpl_dec_u64_locked
448	};
449
450	/** Function table for the NEG instruction. */
451	static const IEMOPUNARYSIZES g_iemAImpl_neg =
452	{
453	iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
454	iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
455	iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
456	iemAImpl_neg_u64, iemAImpl_neg_u64_locked
457	};
458
459	/** Function table for the NOT instruction. */
460	static const IEMOPUNARYSIZES g_iemAImpl_not =
461	{
462	iemAImpl_not_u8, iemAImpl_not_u8_locked,
463	iemAImpl_not_u16, iemAImpl_not_u16_locked,
464	iemAImpl_not_u32, iemAImpl_not_u32_locked,
465	iemAImpl_not_u64, iemAImpl_not_u64_locked
466	};
467
468
469	/** Function table for the ROL instruction. */
470	static const IEMOPSHIFTSIZES g_iemAImpl_rol =
471	{
472	iemAImpl_rol_u8,
473	iemAImpl_rol_u16,
474	iemAImpl_rol_u32,
475	iemAImpl_rol_u64
476	};
477
478	/** Function table for the ROR instruction. */
479	static const IEMOPSHIFTSIZES g_iemAImpl_ror =
480	{
481	iemAImpl_ror_u8,
482	iemAImpl_ror_u16,
483	iemAImpl_ror_u32,
484	iemAImpl_ror_u64
485	};
486
487	/** Function table for the RCL instruction. */
488	static const IEMOPSHIFTSIZES g_iemAImpl_rcl =
489	{
490	iemAImpl_rcl_u8,
491	iemAImpl_rcl_u16,
492	iemAImpl_rcl_u32,
493	iemAImpl_rcl_u64
494	};
495
496	/** Function table for the RCR instruction. */
497	static const IEMOPSHIFTSIZES g_iemAImpl_rcr =
498	{
499	iemAImpl_rcr_u8,
500	iemAImpl_rcr_u16,
501	iemAImpl_rcr_u32,
502	iemAImpl_rcr_u64
503	};
504
505	/** Function table for the SHL instruction. */
506	static const IEMOPSHIFTSIZES g_iemAImpl_shl =
507	{
508	iemAImpl_shl_u8,
509	iemAImpl_shl_u16,
510	iemAImpl_shl_u32,
511	iemAImpl_shl_u64
512	};
513
514	/** Function table for the SHR instruction. */
515	static const IEMOPSHIFTSIZES g_iemAImpl_shr =
516	{
517	iemAImpl_shr_u8,
518	iemAImpl_shr_u16,
519	iemAImpl_shr_u32,
520	iemAImpl_shr_u64
521	};
522
523	/** Function table for the SAR instruction. */
524	static const IEMOPSHIFTSIZES g_iemAImpl_sar =
525	{
526	iemAImpl_sar_u8,
527	iemAImpl_sar_u16,
528	iemAImpl_sar_u32,
529	iemAImpl_sar_u64
530	};
531
532
533	/** Function table for the MUL instruction. */
534	static const IEMOPMULDIVSIZES g_iemAImpl_mul =
535	{
536	iemAImpl_mul_u8,
537	iemAImpl_mul_u16,
538	iemAImpl_mul_u32,
539	iemAImpl_mul_u64
540	};
541
542	/** Function table for the IMUL instruction working implicitly on rAX. */
543	static const IEMOPMULDIVSIZES g_iemAImpl_imul =
544	{
545	iemAImpl_imul_u8,
546	iemAImpl_imul_u16,
547	iemAImpl_imul_u32,
548	iemAImpl_imul_u64
549	};
550
551	/** Function table for the DIV instruction. */
552	static const IEMOPMULDIVSIZES g_iemAImpl_div =
553	{
554	iemAImpl_div_u8,
555	iemAImpl_div_u16,
556	iemAImpl_div_u32,
557	iemAImpl_div_u64
558	};
559
560	/** Function table for the MUL instruction. */
561	static const IEMOPMULDIVSIZES g_iemAImpl_idiv =
562	{
563	iemAImpl_idiv_u8,
564	iemAImpl_idiv_u16,
565	iemAImpl_idiv_u32,
566	iemAImpl_idiv_u64
567	};
568
569	/** Function table for the SHLD instruction */
570	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
571	{
572	iemAImpl_shld_u16,
573	iemAImpl_shld_u32,
574	iemAImpl_shld_u64,
575	};
576
577	/** Function table for the SHRD instruction */
578	static const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
579	{
580	iemAImpl_shrd_u16,
581	iemAImpl_shrd_u32,
582	iemAImpl_shrd_u64,
583	};
584
585
586	/*******************************************************************************
587	* Internal Functions *
588	*******************************************************************************/
589	static VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu);
590	/static VBOXSTRICTRC iemRaiseSelectorNotPresent(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);/
591	static VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
592	static VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr);
593	static VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel);
594	static VBOXSTRICTRC iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr);
595	static VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu);
596	static VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL uSel);
597	static VBOXSTRICTRC iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
598	static VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel);
599	static VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess);
600	static VBOXSTRICTRC iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
601	static VBOXSTRICTRC iemRaiseAlignmentCheckException(PIEMCPU pIemCpu);
602	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
603	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess);
604	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
605	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
606	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
607	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
608	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel);
609	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp);
610	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp);
611	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel);
612	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg);
613
614	#ifdef IEM_VERIFICATION_MODE
615	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu);
616	#endif
617	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue);
618	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue);
619
620
621	/**
622	* Initializes the decoder state.
623	*
624	* @param pIemCpu The per CPU IEM state.
625	*/
626	DECLINLINE(void) iemInitDecoder(PIEMCPU pIemCpu)
627	{
628	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
629
630	pIemCpu->uCpl = CPUMGetGuestCPL(IEMCPU_TO_VMCPU(pIemCpu), CPUMCTX2CORE(pCtx));
631	IEMMODE enmMode = CPUMIsGuestIn64BitCodeEx(pCtx)
632	? IEMMODE_64BIT
633	: pCtx->csHid.Attr.n.u1DefBig /** @todo check if this is correct... */
634	? IEMMODE_32BIT
635	: IEMMODE_16BIT;
636	pIemCpu->enmCpuMode = enmMode;
637	pIemCpu->enmDefAddrMode = enmMode; /** @todo check if this is correct... */
638	pIemCpu->enmEffAddrMode = enmMode;
639	pIemCpu->enmDefOpSize = enmMode; /** @todo check if this is correct... */
640	pIemCpu->enmEffOpSize = enmMode;
641	pIemCpu->fPrefixes = 0;
642	pIemCpu->uRexReg = 0;
643	pIemCpu->uRexB = 0;
644	pIemCpu->uRexIndex = 0;
645	pIemCpu->iEffSeg = X86_SREG_DS;
646	pIemCpu->offOpcode = 0;
647	pIemCpu->cbOpcode = 0;
648	pIemCpu->cActiveMappings = 0;
649	pIemCpu->iNextMapping = 0;
650	}
651
652
653	/**
654	* Prefetch opcodes the first time when starting executing.
655	*
656	* @returns Strict VBox status code.
657	* @param pIemCpu The IEM state.
658	*/
659	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PIEMCPU pIemCpu)
660	{
661	#ifdef IEM_VERIFICATION_MODE
662	uint8_t const cbOldOpcodes = pIemCpu->cbOpcode;
663	#endif
664	iemInitDecoder(pIemCpu);
665
666	/*
667	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
668	*
669	* First translate CS:rIP to a physical address.
670	*/
671	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
672	uint32_t cbToTryRead;
673	RTGCPTR GCPtrPC;
674	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
675	{
676	cbToTryRead = PAGE_SIZE;
677	GCPtrPC = pCtx->rip;
678	if (!IEM_IS_CANONICAL(GCPtrPC))
679	return iemRaiseGeneralProtectionFault0(pIemCpu);
680	cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
681	}
682	else
683	{
684	uint32_t GCPtrPC32 = pCtx->eip;
685	Assert(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
686	if (GCPtrPC32 > pCtx->csHid.u32Limit)
687	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
688	cbToTryRead = pCtx->csHid.u32Limit - GCPtrPC32 + 1;
689	GCPtrPC = pCtx->csHid.u64Base + GCPtrPC32;
690	}
691
692	RTGCPHYS GCPhys;
693	uint64_t fFlags;
694	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrPC, &fFlags, &GCPhys);
695	if (RT_FAILURE(rc))
696	{
697	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
698	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
699	}
700	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
701	{
702	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
703	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
704	}
705	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
706	{
707	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
708	return iemRaisePageFault(pIemCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
709	}
710	GCPhys \|= GCPtrPC & PAGE_OFFSET_MASK;
711	/** @todo Check reserved bits and such stuff. PGM is better at doing
712	* that, so do it when implementing the guest virtual address
713	* TLB... */
714
715	#ifdef IEM_VERIFICATION_MODE
716	/*
717	* Optimistic optimization: Use unconsumed opcode bytes from the previous
718	* instruction.
719	*/
720	/** @todo optimize this differently by not using PGMPhysRead. */
721	RTGCPHYS const offPrevOpcodes = GCPhys - pIemCpu->GCPhysOpcodes;
722	pIemCpu->GCPhysOpcodes = GCPhys;
723	if ( offPrevOpcodes < cbOldOpcodes
724	&& PAGE_SIZE - (GCPhys & PAGE_OFFSET_MASK) > sizeof(pIemCpu->abOpcode))
725	{
726	uint8_t cbNew = cbOldOpcodes - (uint8_t)offPrevOpcodes;
727	memmove(&pIemCpu->abOpcode[0], &pIemCpu->abOpcode[offPrevOpcodes], cbNew);
728	pIemCpu->cbOpcode = cbNew;
729	return VINF_SUCCESS;
730	}
731	#endif
732
733	/*
734	* Read the bytes at this address.
735	*/
736	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
737	if (cbToTryRead > cbLeftOnPage)
738	cbToTryRead = cbLeftOnPage;
739	if (cbToTryRead > sizeof(pIemCpu->abOpcode))
740	cbToTryRead = sizeof(pIemCpu->abOpcode);
741	/** @todo patch manager */
742	if (!pIemCpu->fByPassHandlers)
743	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, pIemCpu->abOpcode, cbToTryRead);
744	else
745	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pIemCpu->abOpcode, GCPhys, cbToTryRead);
746	if (rc != VINF_SUCCESS)
747	{
748	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - read error - rc=%Rrc\n", GCPtrPC, rc));
749	return rc;
750	}
751	pIemCpu->cbOpcode = cbToTryRead;
752
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
759	* exception if it fails.
760	*
761	* @returns Strict VBox status code.
762	* @param pIemCpu The IEM state.
763	* @param cbMin Where to return the opcode byte.
764	*/
765	static VBOXSTRICTRC iemOpcodeFetchMoreBytes(PIEMCPU pIemCpu, size_t cbMin)
766	{
767	/*
768	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
769	*
770	* First translate CS:rIP to a physical address.
771	*/
772	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
773	uint8_t cbLeft = pIemCpu->cbOpcode - pIemCpu->offOpcode; Assert(cbLeft < cbMin);
774	uint32_t cbToTryRead;
775	RTGCPTR GCPtrNext;
776	if (pIemCpu->enmCpuMode == IEMMODE_64BIT)
777	{
778	cbToTryRead = PAGE_SIZE;
779	GCPtrNext = pCtx->rip + pIemCpu->cbOpcode;
780	if (!IEM_IS_CANONICAL(GCPtrNext))
781	return iemRaiseGeneralProtectionFault0(pIemCpu);
782	cbToTryRead = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
783	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
784	}
785	else
786	{
787	uint32_t GCPtrNext32 = pCtx->eip;
788	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pIemCpu->enmCpuMode == IEMMODE_32BIT);
789	GCPtrNext32 += pIemCpu->cbOpcode;
790	if (GCPtrNext32 > pCtx->csHid.u32Limit)
791	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
792	cbToTryRead = pCtx->csHid.u32Limit - GCPtrNext32 + 1;
793	if (cbToTryRead < cbMin - cbLeft)
794	return iemRaiseSelectorBounds(pIemCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
795	GCPtrNext = pCtx->csHid.u64Base + GCPtrNext32;
796	}
797
798	RTGCPHYS GCPhys;
799	uint64_t fFlags;
800	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrNext, &fFlags, &GCPhys);
801	if (RT_FAILURE(rc))
802	{
803	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
804	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
805	}
806	if (!(fFlags & X86_PTE_US) && pIemCpu->uCpl == 3)
807	{
808	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
809	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
810	}
811	if ((fFlags & X86_PTE_PAE_NX) && (pCtx->msrEFER & MSR_K6_EFER_NXE))
812	{
813	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
814	return iemRaisePageFault(pIemCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
815	}
816	GCPhys \|= GCPtrNext & PAGE_OFFSET_MASK;
817	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pIemCpu->cbOpcode));
818	/** @todo Check reserved bits and such stuff. PGM is better at doing
819	* that, so do it when implementing the guest virtual address
820	* TLB... */
821
822	/*
823	* Read the bytes at this address.
824	*/
825	uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
826	if (cbToTryRead > cbLeftOnPage)
827	cbToTryRead = cbLeftOnPage;
828	if (cbToTryRead > sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode)
829	cbToTryRead = sizeof(pIemCpu->abOpcode) - pIemCpu->cbOpcode;
830	Assert(cbToTryRead >= cbMin - cbLeft);
831	if (!pIemCpu->fByPassHandlers)
832	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhys, &pIemCpu->abOpcode[pIemCpu->cbOpcode], cbToTryRead);
833	else
834	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), &pIemCpu->abOpcode[pIemCpu->cbOpcode], GCPhys, cbToTryRead);
835	if (rc != VINF_SUCCESS)
836	{
837	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc\n", GCPtrNext, rc));
838	return rc;
839	}
840	pIemCpu->cbOpcode += cbToTryRead;
841	Log5(("%.*Rhxs\n", pIemCpu->cbOpcode, pIemCpu->abOpcode));
842
843	return VINF_SUCCESS;
844	}
845
846
847	/**
848	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
849	*
850	* @returns Strict VBox status code.
851	* @param pIemCpu The IEM state.
852	* @param pb Where to return the opcode byte.
853	*/
854	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PIEMCPU pIemCpu, uint8_t *pb)
855	{
856	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 1);
857	if (rcStrict == VINF_SUCCESS)
858	{
859	uint8_t offOpcode = pIemCpu->offOpcode;
860	*pb = pIemCpu->abOpcode[offOpcode];
861	pIemCpu->offOpcode = offOpcode + 1;
862	}
863	else
864	*pb = 0;
865	return rcStrict;
866	}
867
868
869	/**
870	* Fetches the next opcode byte.
871	*
872	* @returns Strict VBox status code.
873	* @param pIemCpu The IEM state.
874	* @param pu8 Where to return the opcode byte.
875	*/
876	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PIEMCPU pIemCpu, uint8_t *pu8)
877	{
878	uint8_t const offOpcode = pIemCpu->offOpcode;
879	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
880	return iemOpcodeGetNextU8Slow(pIemCpu, pu8);
881
882	*pu8 = pIemCpu->abOpcode[offOpcode];
883	pIemCpu->offOpcode = offOpcode + 1;
884	return VINF_SUCCESS;
885	}
886
887
888	/**
889	* Fetches the next opcode byte, returns automatically on failure.
890	*
891	* @param a_pu8 Where to return the opcode byte.
892	* @remark Implicitly references pIemCpu.
893	*/
894	#define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
895	do \
896	{ \
897	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pIemCpu, (a_pu8)); \
898	if (rcStrict2 != VINF_SUCCESS) \
899	return rcStrict2; \
900	} while (0)
901
902
903	/**
904	* Fetches the next signed byte from the opcode stream.
905	*
906	* @returns Strict VBox status code.
907	* @param pIemCpu The IEM state.
908	* @param pi8 Where to return the signed byte.
909	*/
910	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PIEMCPU pIemCpu, int8_t *pi8)
911	{
912	return iemOpcodeGetNextU8(pIemCpu, (uint8_t *)pi8);
913	}
914
915
916	/**
917	* Fetches the next signed byte from the opcode stream, returning automatically
918	* on failure.
919	*
920	* @param pi8 Where to return the signed byte.
921	* @remark Implicitly references pIemCpu.
922	*/
923	#define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
924	do \
925	{ \
926	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pIemCpu, (a_pi8)); \
927	if (rcStrict2 != VINF_SUCCESS) \
928	return rcStrict2; \
929	} while (0)
930
931
932	/**
933	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
934	*
935	* @returns Strict VBox status code.
936	* @param pIemCpu The IEM state.
937	* @param pu16 Where to return the opcode dword.
938	*/
939	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
940	{
941	uint8_t u8;
942	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pIemCpu, &u8);
943	if (rcStrict == VINF_SUCCESS)
944	*pu16 = (int8_t)u8;
945	return rcStrict;
946	}
947
948
949	/**
950	* Fetches the next signed byte from the opcode stream, extending it to
951	* unsigned 16-bit.
952	*
953	* @returns Strict VBox status code.
954	* @param pIemCpu The IEM state.
955	* @param pu16 Where to return the unsigned word.
956	*/
957	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PIEMCPU pIemCpu, uint16_t *pu16)
958	{
959	uint8_t const offOpcode = pIemCpu->offOpcode;
960	if (RT_UNLIKELY(offOpcode >= pIemCpu->cbOpcode))
961	return iemOpcodeGetNextS8SxU16Slow(pIemCpu, pu16);
962
963	*pu16 = (int8_t)pIemCpu->abOpcode[offOpcode];
964	pIemCpu->offOpcode = offOpcode + 1;
965	return VINF_SUCCESS;
966	}
967
968
969	/**
970	* Fetches the next signed byte from the opcode stream and sign-extending it to
971	* a word, returning automatically on failure.
972	*
973	* @param pu16 Where to return the word.
974	* @remark Implicitly references pIemCpu.
975	*/
976	#define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
977	do \
978	{ \
979	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pIemCpu, (a_pu16)); \
980	if (rcStrict2 != VINF_SUCCESS) \
981	return rcStrict2; \
982	} while (0)
983
984
985	/**
986	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
987	*
988	* @returns Strict VBox status code.
989	* @param pIemCpu The IEM state.
990	* @param pu16 Where to return the opcode word.
991	*/
992	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PIEMCPU pIemCpu, uint16_t *pu16)
993	{
994	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
995	if (rcStrict == VINF_SUCCESS)
996	{
997	uint8_t offOpcode = pIemCpu->offOpcode;
998	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
999	pIemCpu->offOpcode = offOpcode + 2;
1000	}
1001	else
1002	*pu16 = 0;
1003	return rcStrict;
1004	}
1005
1006
1007	/**
1008	* Fetches the next opcode word.
1009	*
1010	* @returns Strict VBox status code.
1011	* @param pIemCpu The IEM state.
1012	* @param pu16 Where to return the opcode word.
1013	*/
1014	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PIEMCPU pIemCpu, uint16_t *pu16)
1015	{
1016	uint8_t const offOpcode = pIemCpu->offOpcode;
1017	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1018	return iemOpcodeGetNextU16Slow(pIemCpu, pu16);
1019
1020	*pu16 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1021	pIemCpu->offOpcode = offOpcode + 2;
1022	return VINF_SUCCESS;
1023	}
1024
1025
1026	/**
1027	* Fetches the next opcode word, returns automatically on failure.
1028	*
1029	* @param a_pu16 Where to return the opcode word.
1030	* @remark Implicitly references pIemCpu.
1031	*/
1032	#define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
1033	do \
1034	{ \
1035	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pIemCpu, (a_pu16)); \
1036	if (rcStrict2 != VINF_SUCCESS) \
1037	return rcStrict2; \
1038	} while (0)
1039
1040
1041	/**
1042	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1043	*
1044	* @returns Strict VBox status code.
1045	* @param pIemCpu The IEM state.
1046	* @param pu32 Where to return the opcode double word.
1047	*/
1048	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1049	{
1050	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1051	if (rcStrict == VINF_SUCCESS)
1052	{
1053	uint8_t offOpcode = pIemCpu->offOpcode;
1054	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1055	pIemCpu->offOpcode = offOpcode + 2;
1056	}
1057	else
1058	*pu32 = 0;
1059	return rcStrict;
1060	}
1061
1062
1063	/**
1064	* Fetches the next opcode word, zero extending it to a double word.
1065	*
1066	* @returns Strict VBox status code.
1067	* @param pIemCpu The IEM state.
1068	* @param pu32 Where to return the opcode double word.
1069	*/
1070	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PIEMCPU pIemCpu, uint32_t *pu32)
1071	{
1072	uint8_t const offOpcode = pIemCpu->offOpcode;
1073	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1074	return iemOpcodeGetNextU16ZxU32Slow(pIemCpu, pu32);
1075
1076	*pu32 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1077	pIemCpu->offOpcode = offOpcode + 2;
1078	return VINF_SUCCESS;
1079	}
1080
1081
1082	/**
1083	* Fetches the next opcode word and zero extends it to a double word, returns
1084	* automatically on failure.
1085	*
1086	* @param a_pu32 Where to return the opcode double word.
1087	* @remark Implicitly references pIemCpu.
1088	*/
1089	#define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1090	do \
1091	{ \
1092	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pIemCpu, (a_pu32)); \
1093	if (rcStrict2 != VINF_SUCCESS) \
1094	return rcStrict2; \
1095	} while (0)
1096
1097
1098	/**
1099	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1100	*
1101	* @returns Strict VBox status code.
1102	* @param pIemCpu The IEM state.
1103	* @param pu64 Where to return the opcode quad word.
1104	*/
1105	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1106	{
1107	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 2);
1108	if (rcStrict == VINF_SUCCESS)
1109	{
1110	uint8_t offOpcode = pIemCpu->offOpcode;
1111	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1112	pIemCpu->offOpcode = offOpcode + 2;
1113	}
1114	else
1115	*pu64 = 0;
1116	return rcStrict;
1117	}
1118
1119
1120	/**
1121	* Fetches the next opcode word, zero extending it to a quad word.
1122	*
1123	* @returns Strict VBox status code.
1124	* @param pIemCpu The IEM state.
1125	* @param pu64 Where to return the opcode quad word.
1126	*/
1127	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1128	{
1129	uint8_t const offOpcode = pIemCpu->offOpcode;
1130	if (RT_UNLIKELY(offOpcode + 2 > pIemCpu->cbOpcode))
1131	return iemOpcodeGetNextU16ZxU64Slow(pIemCpu, pu64);
1132
1133	*pu64 = RT_MAKE_U16(pIemCpu->abOpcode[offOpcode], pIemCpu->abOpcode[offOpcode + 1]);
1134	pIemCpu->offOpcode = offOpcode + 2;
1135	return VINF_SUCCESS;
1136	}
1137
1138
1139	/**
1140	* Fetches the next opcode word and zero extends it to a quad word, returns
1141	* automatically on failure.
1142	*
1143	* @param a_pu64 Where to return the opcode quad word.
1144	* @remark Implicitly references pIemCpu.
1145	*/
1146	#define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1147	do \
1148	{ \
1149	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pIemCpu, (a_pu64)); \
1150	if (rcStrict2 != VINF_SUCCESS) \
1151	return rcStrict2; \
1152	} while (0)
1153
1154
1155	/**
1156	* Fetches the next signed word from the opcode stream.
1157	*
1158	* @returns Strict VBox status code.
1159	* @param pIemCpu The IEM state.
1160	* @param pi16 Where to return the signed word.
1161	*/
1162	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PIEMCPU pIemCpu, int16_t *pi16)
1163	{
1164	return iemOpcodeGetNextU16(pIemCpu, (uint16_t *)pi16);
1165	}
1166
1167
1168	/**
1169	* Fetches the next signed word from the opcode stream, returning automatically
1170	* on failure.
1171	*
1172	* @param pi16 Where to return the signed word.
1173	* @remark Implicitly references pIemCpu.
1174	*/
1175	#define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1176	do \
1177	{ \
1178	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pIemCpu, (a_pi16)); \
1179	if (rcStrict2 != VINF_SUCCESS) \
1180	return rcStrict2; \
1181	} while (0)
1182
1183
1184	/**
1185	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1186	*
1187	* @returns Strict VBox status code.
1188	* @param pIemCpu The IEM state.
1189	* @param pu32 Where to return the opcode dword.
1190	*/
1191	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PIEMCPU pIemCpu, uint32_t *pu32)
1192	{
1193	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1194	if (rcStrict == VINF_SUCCESS)
1195	{
1196	uint8_t offOpcode = pIemCpu->offOpcode;
1197	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1198	pIemCpu->abOpcode[offOpcode + 1],
1199	pIemCpu->abOpcode[offOpcode + 2],
1200	pIemCpu->abOpcode[offOpcode + 3]);
1201	pIemCpu->offOpcode = offOpcode + 4;
1202	}
1203	else
1204	*pu32 = 0;
1205	return rcStrict;
1206	}
1207
1208
1209	/**
1210	* Fetches the next opcode dword.
1211	*
1212	* @returns Strict VBox status code.
1213	* @param pIemCpu The IEM state.
1214	* @param pu32 Where to return the opcode double word.
1215	*/
1216	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PIEMCPU pIemCpu, uint32_t *pu32)
1217	{
1218	uint8_t const offOpcode = pIemCpu->offOpcode;
1219	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1220	return iemOpcodeGetNextU32Slow(pIemCpu, pu32);
1221
1222	*pu32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1223	pIemCpu->abOpcode[offOpcode + 1],
1224	pIemCpu->abOpcode[offOpcode + 2],
1225	pIemCpu->abOpcode[offOpcode + 3]);
1226	pIemCpu->offOpcode = offOpcode + 4;
1227	return VINF_SUCCESS;
1228	}
1229
1230
1231	/**
1232	* Fetches the next opcode dword, returns automatically on failure.
1233	*
1234	* @param a_pu32 Where to return the opcode dword.
1235	* @remark Implicitly references pIemCpu.
1236	*/
1237	#define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1238	do \
1239	{ \
1240	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pIemCpu, (a_pu32)); \
1241	if (rcStrict2 != VINF_SUCCESS) \
1242	return rcStrict2; \
1243	} while (0)
1244
1245
1246	/**
1247	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1248	*
1249	* @returns Strict VBox status code.
1250	* @param pIemCpu The IEM state.
1251	* @param pu32 Where to return the opcode dword.
1252	*/
1253	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1254	{
1255	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1256	if (rcStrict == VINF_SUCCESS)
1257	{
1258	uint8_t offOpcode = pIemCpu->offOpcode;
1259	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1260	pIemCpu->abOpcode[offOpcode + 1],
1261	pIemCpu->abOpcode[offOpcode + 2],
1262	pIemCpu->abOpcode[offOpcode + 3]);
1263	pIemCpu->offOpcode = offOpcode + 4;
1264	}
1265	else
1266	*pu64 = 0;
1267	return rcStrict;
1268	}
1269
1270
1271	/**
1272	* Fetches the next opcode dword, zero extending it to a quad word.
1273	*
1274	* @returns Strict VBox status code.
1275	* @param pIemCpu The IEM state.
1276	* @param pu64 Where to return the opcode quad word.
1277	*/
1278	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1279	{
1280	uint8_t const offOpcode = pIemCpu->offOpcode;
1281	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1282	return iemOpcodeGetNextU32ZxU64Slow(pIemCpu, pu64);
1283
1284	*pu64 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1285	pIemCpu->abOpcode[offOpcode + 1],
1286	pIemCpu->abOpcode[offOpcode + 2],
1287	pIemCpu->abOpcode[offOpcode + 3]);
1288	pIemCpu->offOpcode = offOpcode + 4;
1289	return VINF_SUCCESS;
1290	}
1291
1292
1293	/**
1294	* Fetches the next opcode dword and zero extends it to a quad word, returns
1295	* automatically on failure.
1296	*
1297	* @param a_pu64 Where to return the opcode quad word.
1298	* @remark Implicitly references pIemCpu.
1299	*/
1300	#define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1301	do \
1302	{ \
1303	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pIemCpu, (a_pu64)); \
1304	if (rcStrict2 != VINF_SUCCESS) \
1305	return rcStrict2; \
1306	} while (0)
1307
1308
1309	/**
1310	* Fetches the next signed double word from the opcode stream.
1311	*
1312	* @returns Strict VBox status code.
1313	* @param pIemCpu The IEM state.
1314	* @param pi32 Where to return the signed double word.
1315	*/
1316	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PIEMCPU pIemCpu, int32_t *pi32)
1317	{
1318	return iemOpcodeGetNextU32(pIemCpu, (uint32_t *)pi32);
1319	}
1320
1321	/**
1322	* Fetches the next signed double word from the opcode stream, returning
1323	* automatically on failure.
1324	*
1325	* @param pi32 Where to return the signed double word.
1326	* @remark Implicitly references pIemCpu.
1327	*/
1328	#define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1329	do \
1330	{ \
1331	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pIemCpu, (a_pi32)); \
1332	if (rcStrict2 != VINF_SUCCESS) \
1333	return rcStrict2; \
1334	} while (0)
1335
1336
1337	/**
1338	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1339	*
1340	* @returns Strict VBox status code.
1341	* @param pIemCpu The IEM state.
1342	* @param pu64 Where to return the opcode qword.
1343	*/
1344	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1345	{
1346	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 4);
1347	if (rcStrict == VINF_SUCCESS)
1348	{
1349	uint8_t offOpcode = pIemCpu->offOpcode;
1350	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1351	pIemCpu->abOpcode[offOpcode + 1],
1352	pIemCpu->abOpcode[offOpcode + 2],
1353	pIemCpu->abOpcode[offOpcode + 3]);
1354	pIemCpu->offOpcode = offOpcode + 4;
1355	}
1356	else
1357	*pu64 = 0;
1358	return rcStrict;
1359	}
1360
1361
1362	/**
1363	* Fetches the next opcode dword, sign extending it into a quad word.
1364	*
1365	* @returns Strict VBox status code.
1366	* @param pIemCpu The IEM state.
1367	* @param pu64 Where to return the opcode quad word.
1368	*/
1369	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64)
1370	{
1371	uint8_t const offOpcode = pIemCpu->offOpcode;
1372	if (RT_UNLIKELY(offOpcode + 4 > pIemCpu->cbOpcode))
1373	return iemOpcodeGetNextS32SxU64Slow(pIemCpu, pu64);
1374
1375	int32_t i32 = RT_MAKE_U32_FROM_U8(pIemCpu->abOpcode[offOpcode],
1376	pIemCpu->abOpcode[offOpcode + 1],
1377	pIemCpu->abOpcode[offOpcode + 2],
1378	pIemCpu->abOpcode[offOpcode + 3]);
1379	*pu64 = i32;
1380	pIemCpu->offOpcode = offOpcode + 4;
1381	return VINF_SUCCESS;
1382	}
1383
1384
1385	/**
1386	* Fetches the next opcode double word and sign extends it to a quad word,
1387	* returns automatically on failure.
1388	*
1389	* @param a_pu64 Where to return the opcode quad word.
1390	* @remark Implicitly references pIemCpu.
1391	*/
1392	#define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1393	do \
1394	{ \
1395	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pIemCpu, (a_pu64)); \
1396	if (rcStrict2 != VINF_SUCCESS) \
1397	return rcStrict2; \
1398	} while (0)
1399
1400
1401	/**
1402	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1403	*
1404	* @returns Strict VBox status code.
1405	* @param pIemCpu The IEM state.
1406	* @param pu64 Where to return the opcode qword.
1407	*/
1408	DECL_NO_INLINE(static, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PIEMCPU pIemCpu, uint64_t *pu64)
1409	{
1410	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pIemCpu, 8);
1411	if (rcStrict == VINF_SUCCESS)
1412	{
1413	uint8_t offOpcode = pIemCpu->offOpcode;
1414	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1415	pIemCpu->abOpcode[offOpcode + 1],
1416	pIemCpu->abOpcode[offOpcode + 2],
1417	pIemCpu->abOpcode[offOpcode + 3],
1418	pIemCpu->abOpcode[offOpcode + 4],
1419	pIemCpu->abOpcode[offOpcode + 5],
1420	pIemCpu->abOpcode[offOpcode + 6],
1421	pIemCpu->abOpcode[offOpcode + 7]);
1422	pIemCpu->offOpcode = offOpcode + 8;
1423	}
1424	else
1425	*pu64 = 0;
1426	return rcStrict;
1427	}
1428
1429
1430	/**
1431	* Fetches the next opcode qword.
1432	*
1433	* @returns Strict VBox status code.
1434	* @param pIemCpu The IEM state.
1435	* @param pu64 Where to return the opcode qword.
1436	*/
1437	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PIEMCPU pIemCpu, uint64_t *pu64)
1438	{
1439	uint8_t const offOpcode = pIemCpu->offOpcode;
1440	if (RT_UNLIKELY(offOpcode + 8 > pIemCpu->cbOpcode))
1441	return iemOpcodeGetNextU64Slow(pIemCpu, pu64);
1442
1443	*pu64 = RT_MAKE_U64_FROM_U8(pIemCpu->abOpcode[offOpcode],
1444	pIemCpu->abOpcode[offOpcode + 1],
1445	pIemCpu->abOpcode[offOpcode + 2],
1446	pIemCpu->abOpcode[offOpcode + 3],
1447	pIemCpu->abOpcode[offOpcode + 4],
1448	pIemCpu->abOpcode[offOpcode + 5],
1449	pIemCpu->abOpcode[offOpcode + 6],
1450	pIemCpu->abOpcode[offOpcode + 7]);
1451	pIemCpu->offOpcode = offOpcode + 8;
1452	return VINF_SUCCESS;
1453	}
1454
1455
1456	/**
1457	* Fetches the next opcode quad word, returns automatically on failure.
1458	*
1459	* @param a_pu64 Where to return the opcode quad word.
1460	* @remark Implicitly references pIemCpu.
1461	*/
1462	#define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1463	do \
1464	{ \
1465	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pIemCpu, (a_pu64)); \
1466	if (rcStrict2 != VINF_SUCCESS) \
1467	return rcStrict2; \
1468	} while (0)
1469
1470
1471	/** @name Misc Worker Functions.
1472	* @{
1473	*/
1474
1475
1476	/**
1477	* Validates a new SS segment.
1478	*
1479	* @returns VBox strict status code.
1480	* @param pIemCpu The IEM per CPU instance data.
1481	* @param pCtx The CPU context.
1482	* @param NewSS The new SS selctor.
1483	* @param uCpl The CPL to load the stack for.
1484	* @param pDesc Where to return the descriptor.
1485	*/
1486	static VBOXSTRICTRC iemMiscValidateNewSS(PIEMCPU pIemCpu, PCCPUMCTX pCtx, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
1487	{
1488	NOREF(pCtx);
1489
1490	/* Null selectors are not allowed (we're not called for dispatching
1491	interrupts with SS=0 in long mode). */
1492	if (!(NewSS & (X86_SEL_MASK \| X86_SEL_LDT)))
1493	{
1494	Log(("iemMiscValidateNewSSandRsp: #x - null selector -> #GP(0)\n", NewSS));
1495	return iemRaiseGeneralProtectionFault0(pIemCpu);
1496	}
1497
1498	/*
1499	* Read the descriptor.
1500	*/
1501	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pIemCpu, pDesc, NewSS);
1502	if (rcStrict != VINF_SUCCESS)
1503	return rcStrict;
1504
1505	/*
1506	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1507	*/
1508	if (!pDesc->Legacy.Gen.u1DescType)
1509	{
1510	Log(("iemMiscValidateNewSSandRsp: %#x - system selector -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1511	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1512	}
1513
1514	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1515	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1516	{
1517	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1518	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1519	}
1520	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1521	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1522	{
1523	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #GP\n", NewSS, pDesc->Legacy.Gen.u4Type));
1524	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1525	}
1526	/** @todo testcase: check if the TSS.ssX RPL is checked. */
1527	if ((NewSS & X86_SEL_RPL) != uCpl)
1528	{
1529	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #GP\n", NewSS, uCpl));
1530	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1531	}
1532	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1533	{
1534	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #GP\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1535	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, NewSS);
1536	}
1537
1538	/* Is it there? */
1539	/** @todo testcase: Is this checked before the canonical / limit check below? */
1540	if (!pDesc->Legacy.Gen.u1Present)
1541	{
1542	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1543	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewSS);
1544	}
1545
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/** @} */
1551
1552	/** @name Raising Exceptions.
1553	*
1554	* @{
1555	*/
1556
1557	/** @name IEM_XCPT_FLAGS_XXX - flags for iemRaiseXcptOrInt.
1558	* @{ */
1559	/** CPU exception. */
1560	#define IEM_XCPT_FLAGS_T_CPU_XCPT RT_BIT_32(0)
1561	/** External interrupt (from PIC, APIC, whatever). */
1562	#define IEM_XCPT_FLAGS_T_EXT_INT RT_BIT_32(1)
1563	/** Software interrupt (int, into or bound). */
1564	#define IEM_XCPT_FLAGS_T_SOFT_INT RT_BIT_32(2)
1565	/** Takes an error code. */
1566	#define IEM_XCPT_FLAGS_ERR RT_BIT_32(3)
1567	/** Takes a CR2. */
1568	#define IEM_XCPT_FLAGS_CR2 RT_BIT_32(4)
1569	/** Generated by the breakpoint instruction. */
1570	#define IEM_XCPT_FLAGS_BP_INSTR RT_BIT_32(5)
1571	/** @} */
1572
1573	/**
1574	* Loads the specified stack far pointer from the TSS.
1575	*
1576	* @returns VBox strict status code.
1577	* @param pIemCpu The IEM per CPU instance data.
1578	* @param pCtx The CPU context.
1579	* @param uCpl The CPL to load the stack for.
1580	* @param pSelSS Where to return the new stack segment.
1581	* @param puEsp Where to return the new stack pointer.
1582	*/
1583	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PIEMCPU pIemCpu, PCCPUMCTX pCtx, uint8_t uCpl,
1584	PRTSEL pSelSS, uint32_t *puEsp)
1585	{
1586	VBOXSTRICTRC rcStrict;
1587	Assert(uCpl < 4);
1588	puEsp = 0; / make gcc happy */
1589	pSelSS = 0; / make gcc happy */
1590
1591	switch (pCtx->trHid.Attr.n.u4Type)
1592	{
1593	/*
1594	* 16-bit TSS (X86TSS16).
1595	*/
1596	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed();
1597	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1598	{
1599	uint32_t off = uCpl * 4 + 2;
1600	if (off + 4 > pCtx->trHid.u32Limit)
1601	{
1602	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->trHid.u32Limit));
1603	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
1604	}
1605
1606	uint32_t u32Tmp = 0; /* gcc maybe... */
1607	rcStrict = iemMemFetchSysU32(pIemCpu, &u32Tmp, UINT8_MAX, pCtx->trHid.u64Base + off);
1608	if (rcStrict == VINF_SUCCESS)
1609	{
1610	*puEsp = RT_LOWORD(u32Tmp);
1611	*pSelSS = RT_HIWORD(u32Tmp);
1612	return VINF_SUCCESS;
1613	}
1614	break;
1615	}
1616
1617	/*
1618	* 32-bit TSS (X86TSS32).
1619	*/
1620	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed();
1621	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1622	{
1623	uint32_t off = uCpl * 8 + 4;
1624	if (off + 7 > pCtx->trHid.u32Limit)
1625	{
1626	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pCtx->trHid.u32Limit));
1627	return iemRaiseTaskSwitchFaultCurrentTSS(pIemCpu);
1628	}
1629
1630	uint64_t u64Tmp;
1631	rcStrict = iemMemFetchSysU64(pIemCpu, &u64Tmp, UINT8_MAX, pCtx->trHid.u64Base + off);
1632	if (rcStrict == VINF_SUCCESS)
1633	{
1634	*puEsp = u64Tmp & UINT32_MAX;
1635	*pSelSS = (RTSEL)(u64Tmp >> 32);
1636	return VINF_SUCCESS;
1637	}
1638	break;
1639	}
1640
1641	default:
1642	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
1643	}
1644	return rcStrict;
1645	}
1646
1647
1648	/**
1649	* Adjust the CPU state according to the exception being raised.
1650	*
1651	* @param pCtx The CPU context.
1652	* @param u8Vector The exception that has been raised.
1653	*/
1654	DECLINLINE(void) iemRaiseXcptAdjustState(PCPUMCTX pCtx, uint8_t u8Vector)
1655	{
1656	switch (u8Vector)
1657	{
1658	case X86_XCPT_DB:
1659	pCtx->dr[7] &= ~X86_DR7_GD;
1660	break;
1661	/** @todo Read the AMD and Intel exception reference... */
1662	}
1663	}
1664
1665
1666	/**
1667	* Implements exceptions and interrupts for real mode.
1668	*
1669	* @returns VBox strict status code.
1670	* @param pIemCpu The IEM per CPU instance data.
1671	* @param pCtx The CPU context.
1672	* @param cbInstr The number of bytes to offset rIP by in the return
1673	* address.
1674	* @param u8Vector The interrupt / exception vector number.
1675	* @param fFlags The flags.
1676	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1677	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1678	*/
1679	static VBOXSTRICTRC
1680	iemRaiseXcptOrIntInRealMode(PIEMCPU pIemCpu,
1681	PCPUMCTX pCtx,
1682	uint8_t cbInstr,
1683	uint8_t u8Vector,
1684	uint32_t fFlags,
1685	uint16_t uErr,
1686	uint64_t uCr2)
1687	{
1688	AssertReturn(pIemCpu->enmCpuMode == IEMMODE_16BIT, VERR_INTERNAL_ERROR_3);
1689	NOREF(uErr); NOREF(uCr2);
1690
1691	/*
1692	* Read the IDT entry.
1693	*/
1694	if (pCtx->idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1695	{
1696	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
1697	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1698	}
1699	RTFAR16 Idte;
1700	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pIemCpu, (uint32_t *)&Idte, UINT8_MAX,
1701	pCtx->idtr.pIdt + UINT32_C(4) * u8Vector);
1702	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1703	return rcStrict;
1704
1705	/*
1706	* Push the stack frame.
1707	*/
1708	uint16_t *pu16Frame;
1709	uint64_t uNewRsp;
1710	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, 6, (void **)&pu16Frame, &uNewRsp);
1711	if (rcStrict != VINF_SUCCESS)
1712	return rcStrict;
1713
1714	pu16Frame[2] = (uint16_t)pCtx->eflags.u;
1715	pu16Frame[1] = (uint16_t)pCtx->cs;
1716	pu16Frame[0] = pCtx->ip + cbInstr;
1717	rcStrict = iemMemStackPushCommitSpecial(pIemCpu, pu16Frame, uNewRsp);
1718	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1719	return rcStrict;
1720
1721	/*
1722	* Load the vector address into cs:ip and make exception specific state
1723	* adjustments.
1724	*/
1725	pCtx->cs = Idte.sel;
1726	pCtx->csHid.u64Base = (uint32_t)Idte.sel << 4;
1727	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1728	pCtx->rip = Idte.off;
1729	pCtx->eflags.Bits.u1IF = 0;
1730
1731	/** @todo do we actually do this in real mode? */
1732	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1733	iemRaiseXcptAdjustState(pCtx, u8Vector);
1734
1735	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1736	}
1737
1738
1739	/**
1740	* Implements exceptions and interrupts for protected mode.
1741	*
1742	* @returns VBox strict status code.
1743	* @param pIemCpu The IEM per CPU instance data.
1744	* @param pCtx The CPU context.
1745	* @param cbInstr The number of bytes to offset rIP by in the return
1746	* address.
1747	* @param u8Vector The interrupt / exception vector number.
1748	* @param fFlags The flags.
1749	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1750	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1751	*/
1752	static VBOXSTRICTRC
1753	iemRaiseXcptOrIntInProtMode(PIEMCPU pIemCpu,
1754	PCPUMCTX pCtx,
1755	uint8_t cbInstr,
1756	uint8_t u8Vector,
1757	uint32_t fFlags,
1758	uint16_t uErr,
1759	uint64_t uCr2)
1760	{
1761	NOREF(cbInstr);
1762
1763	/*
1764	* Read the IDT entry.
1765	*/
1766	if (pCtx->idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
1767	{
1768	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pCtx->idtr.cbIdt));
1769	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1770	}
1771	X86DESC Idte;
1772	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &Idte.u, UINT8_MAX,
1773	pCtx->idtr.pIdt + UINT32_C(8) * u8Vector);
1774	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1775	return rcStrict;
1776	LogFlow(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
1777	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
1778	Idte.Gate.u4ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
1779
1780	/*
1781	* Check the descriptor type, DPL and such.
1782	* ASSUMES this is done in the same order as described for call-gate calls.
1783	*/
1784	if (Idte.Gate.u1DescType)
1785	{
1786	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
1787	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1788	}
1789	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
1790	switch (Idte.Gate.u4Type)
1791	{
1792	case X86_SEL_TYPE_SYS_UNDEFINED:
1793	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
1794	case X86_SEL_TYPE_SYS_LDT:
1795	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1796	case X86_SEL_TYPE_SYS_286_CALL_GATE:
1797	case X86_SEL_TYPE_SYS_UNDEFINED2:
1798	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
1799	case X86_SEL_TYPE_SYS_UNDEFINED3:
1800	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1801	case X86_SEL_TYPE_SYS_386_CALL_GATE:
1802	case X86_SEL_TYPE_SYS_UNDEFINED4:
1803	{
1804	/** @todo check what actually happens when the type is wrong...
1805	* esp. call gates. */
1806	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
1807	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1808	}
1809
1810	case X86_SEL_TYPE_SYS_286_INT_GATE:
1811	case X86_SEL_TYPE_SYS_386_INT_GATE:
1812	fEflToClear \|= X86_EFL_IF;
1813	break;
1814
1815	case X86_SEL_TYPE_SYS_TASK_GATE:
1816	/** @todo task gates. */
1817	AssertFailedReturn(VERR_NOT_SUPPORTED);
1818
1819	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
1820	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
1821	break;
1822
1823	IEM_NOT_REACHED_DEFAULT_CASE_RET();
1824	}
1825
1826	/* Check DPL against CPL if applicable. */
1827	if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
1828	{
1829	if (pIemCpu->uCpl > Idte.Gate.u2Dpl)
1830	{
1831	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pIemCpu->uCpl, Idte.Gate.u2Dpl));
1832	return iemRaiseGeneralProtectionFault(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1833	}
1834	}
1835
1836	/* Is it there? */
1837	if (!Idte.Gate.u1Present)
1838	{
1839	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
1840	return iemRaiseSelectorNotPresentWithErr(pIemCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1841	}
1842
1843	/* A null CS is bad. */
1844	RTSEL NewCS = Idte.Gate.u16Sel;
1845	if (!(NewCS & (X86_SEL_MASK \| X86_SEL_LDT)))
1846	{
1847	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
1848	return iemRaiseGeneralProtectionFault0(pIemCpu);
1849	}
1850
1851	/* Fetch the descriptor for the new CS. */
1852	IEMSELDESC DescCS;
1853	rcStrict = iemMemFetchSelDesc(pIemCpu, &DescCS, NewCS);
1854	if (rcStrict != VINF_SUCCESS)
1855	{
1856	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
1857	return rcStrict;
1858	}
1859
1860	/* Must be a code segment. */
1861	if (!DescCS.Legacy.Gen.u1DescType)
1862	{
1863	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
1864	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1865	}
1866	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
1867	{
1868	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
1869	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1870	}
1871
1872	/* Don't allow lowering the privilege level. */
1873	/** @todo Does the lowering of privileges apply to software interrupts
1874	* only? This has bearings on the more-privileged or
1875	* same-privilege stack behavior further down. A testcase would
1876	* be nice. */
1877	if (DescCS.Legacy.Gen.u2Dpl > pIemCpu->uCpl)
1878	{
1879	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
1880	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
1881	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1882	}
1883	/** @todo is the RPL of the interrupt/trap gate descriptor checked? */
1884
1885	/* Check the new EIP against the new CS limit. */
1886	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
1887	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
1888	? Idte.Gate.u16OffsetLow
1889	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
1890	uint32_t cbLimitCS = X86DESC_LIMIT(DescCS.Legacy);
1891	if (DescCS.Legacy.Gen.u1Granularity)
1892	cbLimitCS = (cbLimitCS << PAGE_SHIFT) \| PAGE_OFFSET_MASK;
1893	if (uNewEip > cbLimitCS)
1894	{
1895	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
1896	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pIemCpu->uCpl));
1897	return iemRaiseGeneralProtectionFault(pIemCpu, NewCS & (X86_SEL_MASK \| X86_SEL_LDT));
1898	}
1899
1900	/* Make sure the selector is present. */
1901	if (!DescCS.Legacy.Gen.u1Present)
1902	{
1903	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
1904	return iemRaiseSelectorNotPresentBySelector(pIemCpu, NewCS);
1905	}
1906
1907	/*
1908	* If the privilege level changes, we need to get a new stack from the TSS.
1909	* This in turns means validating the new SS and ESP...
1910	*/
1911	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
1912	? pIemCpu->uCpl : DescCS.Legacy.Gen.u2Dpl;
1913	if (uNewCpl != pIemCpu->uCpl)
1914	{
1915	RTSEL NewSS;
1916	uint32_t uNewEsp;
1917	rcStrict = iemRaiseLoadStackFromTss32Or16(pIemCpu, pCtx, uNewCpl, &NewSS, &uNewEsp);
1918	if (rcStrict != VINF_SUCCESS)
1919	return rcStrict;
1920
1921	IEMSELDESC DescSS;
1922	rcStrict = iemMiscValidateNewSS(pIemCpu, pCtx, NewSS, uNewCpl, &DescSS);
1923	if (rcStrict != VINF_SUCCESS)
1924	return rcStrict;
1925
1926	/* Check that there is sufficient space for the stack frame. */
1927	uint32_t cbLimitSS = X86DESC_LIMIT(DescSS.Legacy);
1928	if (DescSS.Legacy.Gen.u1Granularity)
1929	cbLimitSS = (cbLimitSS << PAGE_SHIFT) \| PAGE_OFFSET_MASK;
1930	AssertReturn(!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
1931
1932	uint8_t const cbStackFrame = fFlags & IEM_XCPT_FLAGS_ERR ? 24 : 20;
1933	if ( uNewEsp - 1 > cbLimitSS
1934	\|\| uNewEsp < cbStackFrame)
1935	{
1936	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
1937	u8Vector, NewSS, uNewEsp, cbStackFrame));
1938	return iemRaiseSelectorBoundsBySelector(pIemCpu, NewSS);
1939	}
1940
1941	/*
1942	* Start making changes.
1943	*/
1944
1945	/* Create the stack frame. */
1946	RTPTRUNION uStackFrame;
1947	rcStrict = iemMemMap(pIemCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
1948	uNewEsp - cbStackFrame + X86DESC_BASE(DescSS.Legacy), IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
1949	if (rcStrict != VINF_SUCCESS)
1950	return rcStrict;
1951	void * const pvStackFrame = uStackFrame.pv;
1952
1953	if (fFlags & IEM_XCPT_FLAGS_ERR)
1954	*uStackFrame.pu32++ = uErr;
1955	uStackFrame.pu32[0] = (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
1956	? pCtx->eip + cbInstr : pCtx->eip;
1957	uStackFrame.pu32[1] = (pCtx->cs & ~X86_SEL_RPL) \| pIemCpu->uCpl;
1958	uStackFrame.pu32[2] = pCtx->eflags.u;
1959	uStackFrame.pu32[3] = pCtx->esp;
1960	uStackFrame.pu32[4] = pCtx->ss;
1961	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
1962	if (rcStrict != VINF_SUCCESS)
1963	return rcStrict;
1964
1965	/* Mark the selectors 'accessed' (hope this is the correct time). */
1966	/** @todo testcase: excatly _when_ are the accessed bits set - before or
1967	* after pushing the stack frame? (Write protect the gdt + stack to
1968	* find out.) */
1969	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1970	{
1971	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
1972	if (rcStrict != VINF_SUCCESS)
1973	return rcStrict;
1974	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1975	}
1976
1977	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
1978	{
1979	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewSS);
1980	if (rcStrict != VINF_SUCCESS)
1981	return rcStrict;
1982	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
1983	}
1984
1985	/*
1986	* Start commint the register changes (joins with the DPL=CPL branch).
1987	*/
1988	pCtx->ss = NewSS;
1989	pCtx->ssHid.u32Limit = cbLimitSS;
1990	pCtx->ssHid.u64Base = X86DESC_BASE(DescSS.Legacy);
1991	pCtx->ssHid.Attr.u = X86DESC_GET_HID_ATTR(DescSS.Legacy);
1992	pCtx->rsp = uNewEsp - cbStackFrame; /** @todo Is the high word cleared for 16-bit stacks and/or interrupt handlers? */
1993	pIemCpu->uCpl = uNewCpl;
1994	}
1995	/*
1996	* Same privilege, no stack change and smaller stack frame.
1997	*/
1998	else
1999	{
2000	uint64_t uNewRsp;
2001	RTPTRUNION uStackFrame;
2002	uint8_t const cbStackFrame = fFlags & IEM_XCPT_FLAGS_ERR ? 16 : 12;
2003	rcStrict = iemMemStackPushBeginSpecial(pIemCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
2004	if (rcStrict != VINF_SUCCESS)
2005	return rcStrict;
2006	void * const pvStackFrame = uStackFrame.pv;
2007
2008	if (fFlags & IEM_XCPT_FLAGS_ERR)
2009	*uStackFrame.pu32++ = uErr;
2010	uStackFrame.pu32[0] = (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
2011	? pCtx->eip + cbInstr : pCtx->eip;
2012	uStackFrame.pu32[1] = (pCtx->cs & ~X86_SEL_RPL) \| pIemCpu->uCpl;
2013	uStackFrame.pu32[2] = pCtx->eflags.u;
2014	rcStrict = iemMemCommitAndUnmap(pIemCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
2015	if (rcStrict != VINF_SUCCESS)
2016	return rcStrict;
2017
2018	/* Mark the CS selector as 'accessed'. */
2019	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2020	{
2021	rcStrict = iemMemMarkSelDescAccessed(pIemCpu, NewCS);
2022	if (rcStrict != VINF_SUCCESS)
2023	return rcStrict;
2024	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2025	}
2026
2027	/*
2028	* Start committing the register changes (joins with the other branch).
2029	*/
2030	pCtx->rsp = uNewRsp;
2031	}
2032
2033	/* ... register committing continues. */
2034	pCtx->cs = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
2035	pCtx->csHid.u32Limit = cbLimitCS;
2036	pCtx->csHid.u64Base = X86DESC_BASE(DescCS.Legacy);
2037	pCtx->csHid.Attr.u = X86DESC_GET_HID_ATTR(DescCS.Legacy);
2038
2039	pCtx->rip = uNewEip;
2040	pCtx->rflags.u &= ~fEflToClear;
2041
2042	if (fFlags & IEM_XCPT_FLAGS_CR2)
2043	pCtx->cr2 = uCr2;
2044
2045	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
2046	iemRaiseXcptAdjustState(pCtx, u8Vector);
2047
2048	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2049	}
2050
2051
2052	/**
2053	* Implements exceptions and interrupts for V8086 mode.
2054	*
2055	* @returns VBox strict status code.
2056	* @param pIemCpu The IEM per CPU instance data.
2057	* @param pCtx The CPU context.
2058	* @param cbInstr The number of bytes to offset rIP by in the return
2059	* address.
2060	* @param u8Vector The interrupt / exception vector number.
2061	* @param fFlags The flags.
2062	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2063	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2064	*/
2065	static VBOXSTRICTRC
2066	iemRaiseXcptOrIntInV8086Mode(PIEMCPU pIemCpu,
2067	PCPUMCTX pCtx,
2068	uint8_t cbInstr,
2069	uint8_t u8Vector,
2070	uint32_t fFlags,
2071	uint16_t uErr,
2072	uint64_t uCr2)
2073	{
2074	NOREF(pIemCpu); NOREF(pCtx); NOREF(cbInstr); NOREF(u8Vector); NOREF(fFlags); NOREF(uErr); NOREF(uCr2);
2075	AssertMsgFailed(("V8086 exception / interrupt dispatching\n"));
2076	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
2077	}
2078
2079
2080	/**
2081	* Implements exceptions and interrupts for long mode.
2082	*
2083	* @returns VBox strict status code.
2084	* @param pIemCpu The IEM per CPU instance data.
2085	* @param pCtx The CPU context.
2086	* @param cbInstr The number of bytes to offset rIP by in the return
2087	* address.
2088	* @param u8Vector The interrupt / exception vector number.
2089	* @param fFlags The flags.
2090	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2091	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2092	*/
2093	static VBOXSTRICTRC
2094	iemRaiseXcptOrIntInLongMode(PIEMCPU pIemCpu,
2095	PCPUMCTX pCtx,
2096	uint8_t cbInstr,
2097	uint8_t u8Vector,
2098	uint32_t fFlags,
2099	uint16_t uErr,
2100	uint64_t uCr2)
2101	{
2102	NOREF(pIemCpu); NOREF(pCtx); NOREF(cbInstr); NOREF(u8Vector); NOREF(fFlags); NOREF(uErr); NOREF(uCr2);
2103	AssertMsgFailed(("long mode exception / interrupt dispatching\n"));
2104	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
2105	}
2106
2107
2108	/**
2109	* Implements exceptions and interrupts.
2110	*
2111	* All exceptions and interrupts goes thru this function!
2112	*
2113	* @returns VBox strict status code.
2114	* @param pIemCpu The IEM per CPU instance data.
2115	* @param cbInstr The number of bytes to offset rIP by in the return
2116	* address.
2117	* @param u8Vector The interrupt / exception vector number.
2118	* @param fFlags The flags.
2119	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2120	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2121	*/
2122	DECL_NO_INLINE(static, VBOXSTRICTRC)
2123	iemRaiseXcptOrInt(PIEMCPU pIemCpu,
2124	uint8_t cbInstr,
2125	uint8_t u8Vector,
2126	uint32_t fFlags,
2127	uint16_t uErr,
2128	uint64_t uCr2)
2129	{
2130	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2131
2132	/*
2133	* Do recursion accounting.
2134	*/
2135	uint8_t const uPrevXcpt = pIemCpu->uCurXcpt;
2136	uint32_t const fPrevXcpt = pIemCpu->fCurXcpt;
2137	if (pIemCpu->cXcptRecursions == 0)
2138	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
2139	u8Vector, pCtx->cs, pCtx->rip, cbInstr, fFlags, uErr, uCr2));
2140	else
2141	{
2142	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
2143	u8Vector, pCtx->cs, pCtx->rip, cbInstr, fFlags, uErr, uCr2, pIemCpu->uCurXcpt, pIemCpu->cXcptRecursions + 1, fPrevXcpt));
2144
2145	/** @todo double and tripple faults. */
2146	AssertReturn(pIemCpu->cXcptRecursions < 3, VERR_IEM_ASPECT_NOT_IMPLEMENTED);
2147
2148	/** @todo set X86_TRAP_ERR_EXTERNAL when appropriate.
2149	if (fPrevXcpt & IEM_XCPT_FLAGS_T_EXT_INT)
2150	{
2151	....
2152	} */
2153	}
2154	pIemCpu->cXcptRecursions++;
2155	pIemCpu->uCurXcpt = u8Vector;
2156	pIemCpu->fCurXcpt = fFlags;
2157
2158	/*
2159	* Extensive logging.
2160	*/
2161	#ifdef LOG_ENABLED
2162	if (LogIs3Enabled())
2163	{
2164	PVM pVM = IEMCPU_TO_VM(pIemCpu);
2165	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
2166	char szRegs[4096];
2167	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2168	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2169	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2170	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2171	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2172	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2173	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2174	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2175	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2176	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2177	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2178	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2179	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2180	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2181	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2182	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2183	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2184	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2185	" efer=%016VR{efer}\n"
2186	" pat=%016VR{pat}\n"
2187	" sf_mask=%016VR{sf_mask}\n"
2188	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2189	" lstar=%016VR{lstar}\n"
2190	" star=%016VR{star} cstar=%016VR{cstar}\n"
2191	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2192	);
2193
2194	char szInstr[256];
2195	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
2196	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2197	szInstr, sizeof(szInstr), NULL);
2198	Log3(("%s%s\n", szRegs, szInstr));
2199	}
2200	#endif /* LOG_ENABLED */
2201
2202	/*
2203	* Call the mode specific worker function.
2204	*/
2205	VBOXSTRICTRC rcStrict;
2206	if (!(pCtx->cr0 & X86_CR0_PE))
2207	rcStrict = iemRaiseXcptOrIntInRealMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2208	else if (pCtx->msrEFER & MSR_K6_EFER_LMA)
2209	rcStrict = iemRaiseXcptOrIntInLongMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2210	else if (!pCtx->eflags.Bits.u1VM)
2211	rcStrict = iemRaiseXcptOrIntInProtMode( pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2212	else
2213	rcStrict = iemRaiseXcptOrIntInV8086Mode(pIemCpu, pCtx, cbInstr, u8Vector, fFlags, uErr, uCr2);
2214
2215	/*
2216	* Unwind.
2217	*/
2218	pIemCpu->cXcptRecursions--;
2219	pIemCpu->uCurXcpt = uPrevXcpt;
2220	pIemCpu->fCurXcpt = fPrevXcpt;
2221	LogFlow(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv\n",
2222	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pCtx->cs, pCtx->rip, pCtx->ss, pCtx->esp));
2223	return rcStrict;
2224	}
2225
2226
2227	/** \#DE - 00. */
2228	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDivideError(PIEMCPU pIemCpu)
2229	{
2230	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2231	}
2232
2233
2234	/** \#DB - 01. */
2235	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDebugException(PIEMCPU pIemCpu)
2236	{
2237	/** @todo set/clear RF. */
2238	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2239	}
2240
2241
2242	/** \#UD - 06. */
2243	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PIEMCPU pIemCpu)
2244	{
2245	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2246	}
2247
2248
2249	/** \#NM - 07. */
2250	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PIEMCPU pIemCpu)
2251	{
2252	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2253	}
2254
2255
2256	#ifdef SOME_UNUSED_FUNCTION
2257	/** \#TS(err) - 0a. */
2258	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PIEMCPU pIemCpu, uint16_t uErr)
2259	{
2260	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2261	}
2262	#endif
2263
2264
2265	/** \#TS(tr) - 0a. */
2266	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PIEMCPU pIemCpu)
2267	{
2268	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2269	pIemCpu->CTX_SUFF(pCtx)->tr, 0);
2270	}
2271
2272
2273	/** \#NP(err) - 0b. */
2274	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PIEMCPU pIemCpu, uint16_t uErr)
2275	{
2276	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2277	}
2278
2279
2280	/** \#NP(seg) - 0b. */
2281	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySegReg(PIEMCPU pIemCpu, uint32_t iSegReg)
2282	{
2283	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2284	iemSRegFetchU16(pIemCpu, iSegReg) & ~X86_SEL_RPL, 0);
2285	}
2286
2287
2288	/** \#NP(sel) - 0b. */
2289	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
2290	{
2291	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2292	uSel & ~X86_SEL_RPL, 0);
2293	}
2294
2295
2296	/** \#SS(seg) - 0c. */
2297	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PIEMCPU pIemCpu, uint16_t uSel)
2298	{
2299	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2300	uSel & ~X86_SEL_RPL, 0);
2301	}
2302
2303
2304	/** \#GP(n) - 0d. */
2305	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PIEMCPU pIemCpu, uint16_t uErr)
2306	{
2307	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
2308	}
2309
2310
2311	/** \#GP(0) - 0d. */
2312	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PIEMCPU pIemCpu)
2313	{
2314	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2315	}
2316
2317
2318	/** \#GP(sel) - 0d. */
2319	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PIEMCPU pIemCpu, RTSEL Sel)
2320	{
2321	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
2322	Sel & ~X86_SEL_RPL, 0);
2323	}
2324
2325
2326	/** \#GP(0) - 0d. */
2327	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseNotCanonical(PIEMCPU pIemCpu)
2328	{
2329	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2330	}
2331
2332
2333	/** \#GP(sel) - 0d. */
2334	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBounds(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
2335	{
2336	NOREF(iSegReg); NOREF(fAccess);
2337	return iemRaiseXcptOrInt(pIemCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
2338	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2339	}
2340
2341
2342	/** \#GP(sel) - 0d. */
2343	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PIEMCPU pIemCpu, RTSEL Sel)
2344	{
2345	NOREF(Sel);
2346	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2347	}
2348
2349
2350	/** \#GP(sel) - 0d. */
2351	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PIEMCPU pIemCpu, uint32_t iSegReg, uint32_t fAccess)
2352	{
2353	NOREF(iSegReg); NOREF(fAccess);
2354	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
2355	}
2356
2357
2358	/** \#PF(n) - 0e. */
2359	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaisePageFault(PIEMCPU pIemCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
2360	{
2361	uint16_t uErr;
2362	switch (rc)
2363	{
2364	case VERR_PAGE_NOT_PRESENT:
2365	case VERR_PAGE_TABLE_NOT_PRESENT:
2366	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
2367	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
2368	uErr = 0;
2369	break;
2370
2371	default:
2372	AssertMsgFailed(("%Rrc\n", rc));
2373	case VERR_ACCESS_DENIED:
2374	uErr = X86_TRAP_PF_P;
2375	break;
2376
2377	/** @todo reserved */
2378	}
2379
2380	if (pIemCpu->uCpl == 3)
2381	uErr \|= X86_TRAP_PF_US;
2382
2383	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
2384	&& ( (pIemCpu->CTX_SUFF(pCtx)->cr4 & X86_CR4_PAE)
2385	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) ) )
2386	uErr \|= X86_TRAP_PF_ID;
2387
2388	/* Note! RW access callers reporting a WRITE protection fault, will clear
2389	the READ flag before calling. So, read-modify-write accesses (RW)
2390	can safely be reported as READ faults. */
2391	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
2392	uErr \|= X86_TRAP_PF_RW;
2393
2394	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
2395	uErr, GCPtrWhere);
2396	}
2397
2398
2399	/** \#MF(0) - 10. */
2400	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseMathFault(PIEMCPU pIemCpu)
2401	{
2402	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2403	}
2404
2405
2406	/** \#AC(0) - 11. */
2407	DECL_NO_INLINE(static, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PIEMCPU pIemCpu)
2408	{
2409	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2410	}
2411
2412
2413	/**
2414	* Macro for calling iemCImplRaiseDivideError().
2415	*
2416	* This enables us to add/remove arguments and force different levels of
2417	* inlining as we wish.
2418	*
2419	* @return Strict VBox status code.
2420	*/
2421	#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
2422	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
2423	{
2424	NOREF(cbInstr);
2425	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2426	}
2427
2428
2429	/**
2430	* Macro for calling iemCImplRaiseInvalidLockPrefix().
2431	*
2432	* This enables us to add/remove arguments and force different levels of
2433	* inlining as we wish.
2434	*
2435	* @return Strict VBox status code.
2436	*/
2437	#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
2438	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
2439	{
2440	NOREF(cbInstr);
2441	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2442	}
2443
2444
2445	/**
2446	* Macro for calling iemCImplRaiseInvalidOpcode().
2447	*
2448	* This enables us to add/remove arguments and force different levels of
2449	* inlining as we wish.
2450	*
2451	* @return Strict VBox status code.
2452	*/
2453	#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
2454	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
2455	{
2456	NOREF(cbInstr);
2457	return iemRaiseXcptOrInt(pIemCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
2458	}
2459
2460
2461	/** @} */
2462
2463
2464	/*
2465	*
2466	* Helpers routines.
2467	* Helpers routines.
2468	* Helpers routines.
2469	*
2470	*/
2471
2472	/**
2473	* Recalculates the effective operand size.
2474	*
2475	* @param pIemCpu The IEM state.
2476	*/
2477	static void iemRecalEffOpSize(PIEMCPU pIemCpu)
2478	{
2479	switch (pIemCpu->enmCpuMode)
2480	{
2481	case IEMMODE_16BIT:
2482	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
2483	break;
2484	case IEMMODE_32BIT:
2485	pIemCpu->enmEffOpSize = pIemCpu->fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
2486	break;
2487	case IEMMODE_64BIT:
2488	switch (pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
2489	{
2490	case 0:
2491	pIemCpu->enmEffOpSize = pIemCpu->enmDefOpSize;
2492	break;
2493	case IEM_OP_PRF_SIZE_OP:
2494	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
2495	break;
2496	case IEM_OP_PRF_SIZE_REX_W:
2497	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
2498	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
2499	break;
2500	}
2501	break;
2502	default:
2503	AssertFailed();
2504	}
2505	}
2506
2507
2508	/**
2509	* Sets the default operand size to 64-bit and recalculates the effective
2510	* operand size.
2511	*
2512	* @param pIemCpu The IEM state.
2513	*/
2514	static void iemRecalEffOpSize64Default(PIEMCPU pIemCpu)
2515	{
2516	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2517	pIemCpu->enmDefOpSize = IEMMODE_64BIT;
2518	if ((pIemCpu->fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
2519	pIemCpu->enmEffOpSize = IEMMODE_64BIT;
2520	else
2521	pIemCpu->enmEffOpSize = IEMMODE_16BIT;
2522	}
2523
2524
2525	/*
2526	*
2527	* Common opcode decoders.
2528	* Common opcode decoders.
2529	* Common opcode decoders.
2530	*
2531	*/
2532	#include <iprt/mem.h>
2533
2534	/**
2535	* Used to add extra details about a stub case.
2536	* @param pIemCpu The IEM per CPU state.
2537	*/
2538	static void iemOpStubMsg2(PIEMCPU pIemCpu)
2539	{
2540	PVM pVM = IEMCPU_TO_VM(pIemCpu);
2541	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
2542	char szRegs[4096];
2543	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
2544	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
2545	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
2546	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
2547	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
2548	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
2549	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
2550	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
2551	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
2552	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
2553	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
2554	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
2555	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
2556	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
2557	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
2558	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
2559	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
2560	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
2561	" efer=%016VR{efer}\n"
2562	" pat=%016VR{pat}\n"
2563	" sf_mask=%016VR{sf_mask}\n"
2564	"krnl_gs_base=%016VR{krnl_gs_base}\n"
2565	" lstar=%016VR{lstar}\n"
2566	" star=%016VR{star} cstar=%016VR{cstar}\n"
2567	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
2568	);
2569
2570	char szInstr[256];
2571	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
2572	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
2573	szInstr, sizeof(szInstr), NULL);
2574
2575	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
2576	}
2577
2578
2579	/** Stubs an opcode. */
2580	#define FNIEMOP_STUB(a_Name) \
2581	FNIEMOP_DEF(a_Name) \
2582	{ \
2583	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
2584	iemOpStubMsg2(pIemCpu); \
2585	RTAssertPanic(); \
2586	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
2587	} \
2588	typedef int ignore_semicolon
2589
2590	/** Stubs an opcode. */
2591	#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
2592	FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
2593	{ \
2594	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
2595	iemOpStubMsg2(pIemCpu); \
2596	RTAssertPanic(); \
2597	NOREF(a_Name0); \
2598	return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
2599	} \
2600	typedef int ignore_semicolon
2601
2602
2603
2604	/** @name Register Access.
2605	* @{
2606	*/
2607
2608	/**
2609	* Gets a reference (pointer) to the specified hidden segment register.
2610	*
2611	* @returns Hidden register reference.
2612	* @param pIemCpu The per CPU data.
2613	* @param iSegReg The segment register.
2614	*/
2615	static PCPUMSELREGHID iemSRegGetHid(PIEMCPU pIemCpu, uint8_t iSegReg)
2616	{
2617	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2618	switch (iSegReg)
2619	{
2620	case X86_SREG_ES: return &pCtx->esHid;
2621	case X86_SREG_CS: return &pCtx->csHid;
2622	case X86_SREG_SS: return &pCtx->ssHid;
2623	case X86_SREG_DS: return &pCtx->dsHid;
2624	case X86_SREG_FS: return &pCtx->fsHid;
2625	case X86_SREG_GS: return &pCtx->gsHid;
2626	}
2627	AssertFailedReturn(NULL);
2628	}
2629
2630
2631	/**
2632	* Gets a reference (pointer) to the specified segment register (the selector
2633	* value).
2634	*
2635	* @returns Pointer to the selector variable.
2636	* @param pIemCpu The per CPU data.
2637	* @param iSegReg The segment register.
2638	*/
2639	static uint16_t *iemSRegRef(PIEMCPU pIemCpu, uint8_t iSegReg)
2640	{
2641	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2642	switch (iSegReg)
2643	{
2644	case X86_SREG_ES: return &pCtx->es;
2645	case X86_SREG_CS: return &pCtx->cs;
2646	case X86_SREG_SS: return &pCtx->ss;
2647	case X86_SREG_DS: return &pCtx->ds;
2648	case X86_SREG_FS: return &pCtx->fs;
2649	case X86_SREG_GS: return &pCtx->gs;
2650	}
2651	AssertFailedReturn(NULL);
2652	}
2653
2654
2655	/**
2656	* Fetches the selector value of a segment register.
2657	*
2658	* @returns The selector value.
2659	* @param pIemCpu The per CPU data.
2660	* @param iSegReg The segment register.
2661	*/
2662	static uint16_t iemSRegFetchU16(PIEMCPU pIemCpu, uint8_t iSegReg)
2663	{
2664	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2665	switch (iSegReg)
2666	{
2667	case X86_SREG_ES: return pCtx->es;
2668	case X86_SREG_CS: return pCtx->cs;
2669	case X86_SREG_SS: return pCtx->ss;
2670	case X86_SREG_DS: return pCtx->ds;
2671	case X86_SREG_FS: return pCtx->fs;
2672	case X86_SREG_GS: return pCtx->gs;
2673	}
2674	AssertFailedReturn(0xffff);
2675	}
2676
2677
2678	/**
2679	* Gets a reference (pointer) to the specified general register.
2680	*
2681	* @returns Register reference.
2682	* @param pIemCpu The per CPU data.
2683	* @param iReg The general register.
2684	*/
2685	static void *iemGRegRef(PIEMCPU pIemCpu, uint8_t iReg)
2686	{
2687	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2688	switch (iReg)
2689	{
2690	case X86_GREG_xAX: return &pCtx->rax;
2691	case X86_GREG_xCX: return &pCtx->rcx;
2692	case X86_GREG_xDX: return &pCtx->rdx;
2693	case X86_GREG_xBX: return &pCtx->rbx;
2694	case X86_GREG_xSP: return &pCtx->rsp;
2695	case X86_GREG_xBP: return &pCtx->rbp;
2696	case X86_GREG_xSI: return &pCtx->rsi;
2697	case X86_GREG_xDI: return &pCtx->rdi;
2698	case X86_GREG_x8: return &pCtx->r8;
2699	case X86_GREG_x9: return &pCtx->r9;
2700	case X86_GREG_x10: return &pCtx->r10;
2701	case X86_GREG_x11: return &pCtx->r11;
2702	case X86_GREG_x12: return &pCtx->r12;
2703	case X86_GREG_x13: return &pCtx->r13;
2704	case X86_GREG_x14: return &pCtx->r14;
2705	case X86_GREG_x15: return &pCtx->r15;
2706	}
2707	AssertFailedReturn(NULL);
2708	}
2709
2710
2711	/**
2712	* Gets a reference (pointer) to the specified 8-bit general register.
2713	*
2714	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
2715	*
2716	* @returns Register reference.
2717	* @param pIemCpu The per CPU data.
2718	* @param iReg The register.
2719	*/
2720	static uint8_t *iemGRegRefU8(PIEMCPU pIemCpu, uint8_t iReg)
2721	{
2722	if (pIemCpu->fPrefixes & IEM_OP_PRF_REX)
2723	return (uint8_t *)iemGRegRef(pIemCpu, iReg);
2724
2725	uint8_t pu8Reg = (uint8_t )iemGRegRef(pIemCpu, iReg & 3);
2726	if (iReg >= 4)
2727	pu8Reg++;
2728	return pu8Reg;
2729	}
2730
2731
2732	/**
2733	* Fetches the value of a 8-bit general register.
2734	*
2735	* @returns The register value.
2736	* @param pIemCpu The per CPU data.
2737	* @param iReg The register.
2738	*/
2739	static uint8_t iemGRegFetchU8(PIEMCPU pIemCpu, uint8_t iReg)
2740	{
2741	uint8_t const *pbSrc = iemGRegRefU8(pIemCpu, iReg);
2742	return *pbSrc;
2743	}
2744
2745
2746	/**
2747	* Fetches the value of a 16-bit general register.
2748	*
2749	* @returns The register value.
2750	* @param pIemCpu The per CPU data.
2751	* @param iReg The register.
2752	*/
2753	static uint16_t iemGRegFetchU16(PIEMCPU pIemCpu, uint8_t iReg)
2754	{
2755	return (uint16_t )iemGRegRef(pIemCpu, iReg);
2756	}
2757
2758
2759	/**
2760	* Fetches the value of a 32-bit general register.
2761	*
2762	* @returns The register value.
2763	* @param pIemCpu The per CPU data.
2764	* @param iReg The register.
2765	*/
2766	static uint32_t iemGRegFetchU32(PIEMCPU pIemCpu, uint8_t iReg)
2767	{
2768	return (uint32_t )iemGRegRef(pIemCpu, iReg);
2769	}
2770
2771
2772	/**
2773	* Fetches the value of a 64-bit general register.
2774	*
2775	* @returns The register value.
2776	* @param pIemCpu The per CPU data.
2777	* @param iReg The register.
2778	*/
2779	static uint64_t iemGRegFetchU64(PIEMCPU pIemCpu, uint8_t iReg)
2780	{
2781	return (uint64_t )iemGRegRef(pIemCpu, iReg);
2782	}
2783
2784
2785	/**
2786	* Is the FPU state in FXSAVE format or not.
2787	*
2788	* @returns true if it is, false if it's in FNSAVE.
2789	* @param pVCpu The virtual CPU handle.
2790	*/
2791	DECLINLINE(bool) iemFRegIsFxSaveFormat(PIEMCPU pIemCpu)
2792	{
2793	#ifdef RT_ARCH_AMD64
2794	NOREF(pIemCpu);
2795	return true;
2796	#else
2797	NOREF(pIemCpu); /// @todo return pVCpu->pVMR3->cpum.s.CPUFeatures.edx.u1FXSR;
2798	return true;
2799	#endif
2800	}
2801
2802
2803	/**
2804	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
2805	*
2806	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2807	* segment limit.
2808	*
2809	* @param pIemCpu The per CPU data.
2810	* @param offNextInstr The offset of the next instruction.
2811	*/
2812	static VBOXSTRICTRC iemRegRipRelativeJumpS8(PIEMCPU pIemCpu, int8_t offNextInstr)
2813	{
2814	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2815	switch (pIemCpu->enmEffOpSize)
2816	{
2817	case IEMMODE_16BIT:
2818	{
2819	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
2820	if ( uNewIp > pCtx->csHid.u32Limit
2821	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2822	return iemRaiseGeneralProtectionFault0(pIemCpu);
2823	pCtx->rip = uNewIp;
2824	break;
2825	}
2826
2827	case IEMMODE_32BIT:
2828	{
2829	Assert(pCtx->rip <= UINT32_MAX);
2830	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2831
2832	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
2833	if (uNewEip > pCtx->csHid.u32Limit)
2834	return iemRaiseGeneralProtectionFault0(pIemCpu);
2835	pCtx->rip = uNewEip;
2836	break;
2837	}
2838
2839	case IEMMODE_64BIT:
2840	{
2841	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2842
2843	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
2844	if (!IEM_IS_CANONICAL(uNewRip))
2845	return iemRaiseGeneralProtectionFault0(pIemCpu);
2846	pCtx->rip = uNewRip;
2847	break;
2848	}
2849
2850	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2851	}
2852
2853	return VINF_SUCCESS;
2854	}
2855
2856
2857	/**
2858	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
2859	*
2860	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2861	* segment limit.
2862	*
2863	* @returns Strict VBox status code.
2864	* @param pIemCpu The per CPU data.
2865	* @param offNextInstr The offset of the next instruction.
2866	*/
2867	static VBOXSTRICTRC iemRegRipRelativeJumpS16(PIEMCPU pIemCpu, int16_t offNextInstr)
2868	{
2869	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2870	Assert(pIemCpu->enmEffOpSize == IEMMODE_16BIT);
2871
2872	uint16_t uNewIp = pCtx->ip + offNextInstr + pIemCpu->offOpcode;
2873	if ( uNewIp > pCtx->csHid.u32Limit
2874	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2875	return iemRaiseGeneralProtectionFault0(pIemCpu);
2876	/** @todo Test 16-bit jump in 64-bit mode. */
2877	pCtx->rip = uNewIp;
2878
2879	return VINF_SUCCESS;
2880	}
2881
2882
2883	/**
2884	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
2885	*
2886	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2887	* segment limit.
2888	*
2889	* @returns Strict VBox status code.
2890	* @param pIemCpu The per CPU data.
2891	* @param offNextInstr The offset of the next instruction.
2892	*/
2893	static VBOXSTRICTRC iemRegRipRelativeJumpS32(PIEMCPU pIemCpu, int32_t offNextInstr)
2894	{
2895	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2896	Assert(pIemCpu->enmEffOpSize != IEMMODE_16BIT);
2897
2898	if (pIemCpu->enmEffOpSize == IEMMODE_32BIT)
2899	{
2900	Assert(pCtx->rip <= UINT32_MAX); Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2901
2902	uint32_t uNewEip = pCtx->eip + offNextInstr + pIemCpu->offOpcode;
2903	if (uNewEip > pCtx->csHid.u32Limit)
2904	return iemRaiseGeneralProtectionFault0(pIemCpu);
2905	pCtx->rip = uNewEip;
2906	}
2907	else
2908	{
2909	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2910
2911	uint64_t uNewRip = pCtx->rip + offNextInstr + pIemCpu->offOpcode;
2912	if (!IEM_IS_CANONICAL(uNewRip))
2913	return iemRaiseGeneralProtectionFault0(pIemCpu);
2914	pCtx->rip = uNewRip;
2915	}
2916	return VINF_SUCCESS;
2917	}
2918
2919
2920	/**
2921	* Performs a near jump to the specified address.
2922	*
2923	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2924	* segment limit.
2925	*
2926	* @param pIemCpu The per CPU data.
2927	* @param uNewRip The new RIP value.
2928	*/
2929	static VBOXSTRICTRC iemRegRipJump(PIEMCPU pIemCpu, uint64_t uNewRip)
2930	{
2931	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2932	switch (pIemCpu->enmEffOpSize)
2933	{
2934	case IEMMODE_16BIT:
2935	{
2936	Assert(uNewRip <= UINT16_MAX);
2937	if ( uNewRip > pCtx->csHid.u32Limit
2938	&& pIemCpu->enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
2939	return iemRaiseGeneralProtectionFault0(pIemCpu);
2940	/** @todo Test 16-bit jump in 64-bit mode. */
2941	pCtx->rip = uNewRip;
2942	break;
2943	}
2944
2945	case IEMMODE_32BIT:
2946	{
2947	Assert(uNewRip <= UINT32_MAX);
2948	Assert(pCtx->rip <= UINT32_MAX);
2949	Assert(pIemCpu->enmCpuMode != IEMMODE_64BIT);
2950
2951	if (uNewRip > pCtx->csHid.u32Limit)
2952	return iemRaiseGeneralProtectionFault0(pIemCpu);
2953	pCtx->rip = uNewRip;
2954	break;
2955	}
2956
2957	case IEMMODE_64BIT:
2958	{
2959	Assert(pIemCpu->enmCpuMode == IEMMODE_64BIT);
2960
2961	if (!IEM_IS_CANONICAL(uNewRip))
2962	return iemRaiseGeneralProtectionFault0(pIemCpu);
2963	pCtx->rip = uNewRip;
2964	break;
2965	}
2966
2967	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2968	}
2969
2970	return VINF_SUCCESS;
2971	}
2972
2973
2974	/**
2975	* Get the address of the top of the stack.
2976	*
2977	* @param pCtx The CPU context which SP/ESP/RSP should be
2978	* read.
2979	*/
2980	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCCPUMCTX pCtx)
2981	{
2982	if (pCtx->ssHid.Attr.n.u1Long)
2983	return pCtx->rsp;
2984	if (pCtx->ssHid.Attr.n.u1DefBig)
2985	return pCtx->esp;
2986	return pCtx->sp;
2987	}
2988
2989
2990	/**
2991	* Updates the RIP/EIP/IP to point to the next instruction.
2992	*
2993	* @param pIemCpu The per CPU data.
2994	* @param cbInstr The number of bytes to add.
2995	*/
2996	static void iemRegAddToRip(PIEMCPU pIemCpu, uint8_t cbInstr)
2997	{
2998	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
2999	switch (pIemCpu->enmCpuMode)
3000	{
3001	case IEMMODE_16BIT:
3002	Assert(pCtx->rip <= UINT16_MAX);
3003	pCtx->eip += cbInstr;
3004	pCtx->eip &= UINT32_C(0xffff);
3005	break;
3006
3007	case IEMMODE_32BIT:
3008	pCtx->eip += cbInstr;
3009	Assert(pCtx->rip <= UINT32_MAX);
3010	break;
3011
3012	case IEMMODE_64BIT:
3013	pCtx->rip += cbInstr;
3014	break;
3015	default: AssertFailed();
3016	}
3017	}
3018
3019
3020	/**
3021	* Updates the RIP/EIP/IP to point to the next instruction.
3022	*
3023	* @param pIemCpu The per CPU data.
3024	*/
3025	static void iemRegUpdateRip(PIEMCPU pIemCpu)
3026	{
3027	return iemRegAddToRip(pIemCpu, pIemCpu->offOpcode);
3028	}
3029
3030
3031	/**
3032	* Adds to the stack pointer.
3033	*
3034	* @param pCtx The CPU context which SP/ESP/RSP should be
3035	* updated.
3036	* @param cbToAdd The number of bytes to add.
3037	*/
3038	DECLINLINE(void) iemRegAddToRsp(PCPUMCTX pCtx, uint8_t cbToAdd)
3039	{
3040	if (pCtx->ssHid.Attr.n.u1Long)
3041	pCtx->rsp += cbToAdd;
3042	else if (pCtx->ssHid.Attr.n.u1DefBig)
3043	pCtx->esp += cbToAdd;
3044	else
3045	pCtx->sp += cbToAdd;
3046	}
3047
3048
3049	/**
3050	* Subtracts from the stack pointer.
3051	*
3052	* @param pCtx The CPU context which SP/ESP/RSP should be
3053	* updated.
3054	* @param cbToSub The number of bytes to subtract.
3055	*/
3056	DECLINLINE(void) iemRegSubFromRsp(PCPUMCTX pCtx, uint8_t cbToSub)
3057	{
3058	if (pCtx->ssHid.Attr.n.u1Long)
3059	pCtx->rsp -= cbToSub;
3060	else if (pCtx->ssHid.Attr.n.u1DefBig)
3061	pCtx->esp -= cbToSub;
3062	else
3063	pCtx->sp -= cbToSub;
3064	}
3065
3066
3067	/**
3068	* Adds to the temporary stack pointer.
3069	*
3070	* @param pTmpRsp The temporary SP/ESP/RSP to update.
3071	* @param cbToAdd The number of bytes to add.
3072	* @param pCtx Where to get the current stack mode.
3073	*/
3074	DECLINLINE(void) iemRegAddToRspEx(PRTUINT64U pTmpRsp, uint8_t cbToAdd, PCCPUMCTX pCtx)
3075	{
3076	if (pCtx->ssHid.Attr.n.u1Long)
3077	pTmpRsp->u += cbToAdd;
3078	else if (pCtx->ssHid.Attr.n.u1DefBig)
3079	pTmpRsp->DWords.dw0 += cbToAdd;
3080	else
3081	pTmpRsp->Words.w0 += cbToAdd;
3082	}
3083
3084
3085	/**
3086	* Subtracts from the temporary stack pointer.
3087	*
3088	* @param pTmpRsp The temporary SP/ESP/RSP to update.
3089	* @param cbToSub The number of bytes to subtract.
3090	* @param pCtx Where to get the current stack mode.
3091	*/
3092	DECLINLINE(void) iemRegSubFromRspEx(PRTUINT64U pTmpRsp, uint8_t cbToSub, PCCPUMCTX pCtx)
3093	{
3094	if (pCtx->ssHid.Attr.n.u1Long)
3095	pTmpRsp->u -= cbToSub;
3096	else if (pCtx->ssHid.Attr.n.u1DefBig)
3097	pTmpRsp->DWords.dw0 -= cbToSub;
3098	else
3099	pTmpRsp->Words.w0 -= cbToSub;
3100	}
3101
3102
3103	/**
3104	* Calculates the effective stack address for a push of the specified size as
3105	* well as the new RSP value (upper bits may be masked).
3106	*
3107	* @returns Effective stack addressf for the push.
3108	* @param pCtx Where to get the current stack mode.
3109	* @param cbItem The size of the stack item to pop.
3110	* @param puNewRsp Where to return the new RSP value.
3111	*/
3112	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
3113	{
3114	RTUINT64U uTmpRsp;
3115	RTGCPTR GCPtrTop;
3116	uTmpRsp.u = pCtx->rsp;
3117
3118	if (pCtx->ssHid.Attr.n.u1Long)
3119	GCPtrTop = uTmpRsp.u -= cbItem;
3120	else if (pCtx->ssHid.Attr.n.u1DefBig)
3121	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
3122	else
3123	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
3124	*puNewRsp = uTmpRsp.u;
3125	return GCPtrTop;
3126	}
3127
3128
3129	/**
3130	* Gets the current stack pointer and calculates the value after a pop of the
3131	* specified size.
3132	*
3133	* @returns Current stack pointer.
3134	* @param pCtx Where to get the current stack mode.
3135	* @param cbItem The size of the stack item to pop.
3136	* @param puNewRsp Where to return the new RSP value.
3137	*/
3138	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCCPUMCTX pCtx, uint8_t cbItem, uint64_t *puNewRsp)
3139	{
3140	RTUINT64U uTmpRsp;
3141	RTGCPTR GCPtrTop;
3142	uTmpRsp.u = pCtx->rsp;
3143
3144	if (pCtx->ssHid.Attr.n.u1Long)
3145	{
3146	GCPtrTop = uTmpRsp.u;
3147	uTmpRsp.u += cbItem;
3148	}
3149	else if (pCtx->ssHid.Attr.n.u1DefBig)
3150	{
3151	GCPtrTop = uTmpRsp.DWords.dw0;
3152	uTmpRsp.DWords.dw0 += cbItem;
3153	}
3154	else
3155	{
3156	GCPtrTop = uTmpRsp.Words.w0;
3157	uTmpRsp.Words.w0 += cbItem;
3158	}
3159	*puNewRsp = uTmpRsp.u;
3160	return GCPtrTop;
3161	}
3162
3163
3164	/**
3165	* Calculates the effective stack address for a push of the specified size as
3166	* well as the new temporary RSP value (upper bits may be masked).
3167	*
3168	* @returns Effective stack addressf for the push.
3169	* @param pTmpRsp The temporary stack pointer. This is updated.
3170	* @param cbItem The size of the stack item to pop.
3171	* @param puNewRsp Where to return the new RSP value.
3172	*/
3173	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PRTUINT64U pTmpRsp, uint8_t cbItem, PCCPUMCTX pCtx)
3174	{
3175	RTGCPTR GCPtrTop;
3176
3177	if (pCtx->ssHid.Attr.n.u1Long)
3178	GCPtrTop = pTmpRsp->u -= cbItem;
3179	else if (pCtx->ssHid.Attr.n.u1DefBig)
3180	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
3181	else
3182	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
3183	return GCPtrTop;
3184	}
3185
3186
3187	/**
3188	* Gets the effective stack address for a pop of the specified size and
3189	* calculates and updates the temporary RSP.
3190	*
3191	* @returns Current stack pointer.
3192	* @param pTmpRsp The temporary stack pointer. This is updated.
3193	* @param pCtx Where to get the current stack mode.
3194	* @param cbItem The size of the stack item to pop.
3195	*/
3196	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PRTUINT64U pTmpRsp, uint8_t cbItem, PCCPUMCTX pCtx)
3197	{
3198	RTGCPTR GCPtrTop;
3199	if (pCtx->ssHid.Attr.n.u1Long)
3200	{
3201	GCPtrTop = pTmpRsp->u;
3202	pTmpRsp->u += cbItem;
3203	}
3204	else if (pCtx->ssHid.Attr.n.u1DefBig)
3205	{
3206	GCPtrTop = pTmpRsp->DWords.dw0;
3207	pTmpRsp->DWords.dw0 += cbItem;
3208	}
3209	else
3210	{
3211	GCPtrTop = pTmpRsp->Words.w0;
3212	pTmpRsp->Words.w0 += cbItem;
3213	}
3214	return GCPtrTop;
3215	}
3216
3217
3218	/**
3219	* Checks if an Intel CPUID feature bit is set.
3220	*
3221	* @returns true / false.
3222	*
3223	* @param pIemCpu The IEM per CPU data.
3224	* @param fEdx The EDX bit to test, or 0 if ECX.
3225	* @param fEcx The ECX bit to test, or 0 if EDX.
3226	* @remarks Used via IEM_IS_INTEL_CPUID_FEATURE_PRESENT_EDX,
3227	* IEM_IS_INTEL_CPUID_FEATURE_PRESENT_ECX and others.
3228	*/
3229	static bool iemRegIsIntelCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
3230	{
3231	uint32_t uEax, uEbx, uEcx, uEdx;
3232	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x00000001, &uEax, &uEbx, &uEcx, &uEdx);
3233	return (fEcx && (uEcx & fEcx))
3234	\|\| (fEdx && (uEdx & fEdx));
3235	}
3236
3237
3238	/**
3239	* Checks if an AMD CPUID feature bit is set.
3240	*
3241	* @returns true / false.
3242	*
3243	* @param pIemCpu The IEM per CPU data.
3244	* @param fEdx The EDX bit to test, or 0 if ECX.
3245	* @param fEcx The ECX bit to test, or 0 if EDX.
3246	* @remarks Used via IEM_IS_AMD_CPUID_FEATURE_PRESENT_EDX,
3247	* IEM_IS_AMD_CPUID_FEATURE_PRESENT_ECX and others.
3248	*/
3249	static bool iemRegIsAmdCpuIdFeaturePresent(PIEMCPU pIemCpu, uint32_t fEdx, uint32_t fEcx)
3250	{
3251	uint32_t uEax, uEbx, uEcx, uEdx;
3252	CPUMGetGuestCpuId(IEMCPU_TO_VMCPU(pIemCpu), 0x80000001, &uEax, &uEbx, &uEcx, &uEdx);
3253	return (fEcx && (uEcx & fEcx))
3254	\|\| (fEdx && (uEdx & fEdx));
3255	}
3256
3257	/** @} */
3258
3259
3260	/** @name FPU access and helpers.
3261	*
3262	* @{
3263	*/
3264
3265
3266	/**
3267	* Hook for preparing to use the host FPU.
3268	*
3269	* This is necessary in ring-0 and raw-mode context.
3270	*
3271	* @param pIemCpu The IEM per CPU data.
3272	*/
3273	DECLINLINE(void) iemFpuPrepareUsage(PIEMCPU pIemCpu)
3274	{
3275	#ifdef IN_RING3
3276	NOREF(pIemCpu);
3277	#else
3278	# error "Implement me"
3279	#endif
3280	}
3281
3282
3283	/**
3284	* Stores a QNaN value into a FPU register.
3285	*
3286	* @param pReg Pointer to the register.
3287	*/
3288	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
3289	{
3290	pReg->au32[0] = UINT32_C(0x00000000);
3291	pReg->au32[1] = UINT32_C(0xc0000000);
3292	pReg->au16[4] = UINT16_C(0xffff);
3293	}
3294
3295
3296	/**
3297	* Updates the FOP, FPU.CS and FPUIP registers.
3298	*
3299	* @param pIemCpu The IEM per CPU data.
3300	* @param pCtx The CPU context.
3301	*/
3302	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PIEMCPU pIemCpu, PCPUMCTX pCtx)
3303	{
3304	pCtx->fpu.FOP = pIemCpu->abOpcode[pIemCpu->offFpuOpcode]
3305	\| ((uint16_t)(pIemCpu->abOpcode[pIemCpu->offFpuOpcode - 1] & 0x7) << 8);
3306	/** @todo FPU.CS and FPUIP needs to be kept seperately. */
3307	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu))
3308	{
3309	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
3310	* happens in real mode here based on the fnsave and fnstenv images. */
3311	pCtx->fpu.CS = 0;
3312	pCtx->fpu.FPUIP = pCtx->eip \| ((uint32_t)pCtx->cs << 4);
3313	}
3314	else
3315	{
3316	pCtx->fpu.CS = pCtx->cs;
3317	pCtx->fpu.FPUIP = pCtx->rip;
3318	}
3319	}
3320
3321
3322	/**
3323	* Updates the FPU.DS and FPUDP registers.
3324	*
3325	* @param pIemCpu The IEM per CPU data.
3326	* @param pCtx The CPU context.
3327	* @param iEffSeg The effective segment register.
3328	* @param GCPtrEff The effective address relative to @a iEffSeg.
3329	*/
3330	DECLINLINE(void) iemFpuUpdateDP(PIEMCPU pIemCpu, PCPUMCTX pCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3331	{
3332	RTSEL sel;
3333	switch (iEffSeg)
3334	{
3335	case X86_SREG_DS: sel = pCtx->ds; break;
3336	case X86_SREG_SS: sel = pCtx->ss; break;
3337	case X86_SREG_CS: sel = pCtx->cs; break;
3338	case X86_SREG_ES: sel = pCtx->es; break;
3339	case X86_SREG_FS: sel = pCtx->fs; break;
3340	case X86_SREG_GS: sel = pCtx->gs; break;
3341	default:
3342	AssertMsgFailed(("%d\n", iEffSeg));
3343	sel = pCtx->ds;
3344	}
3345	/** @todo FPU.DS and FPUDP needs to be kept seperately. */
3346	if (IEM_IS_REAL_OR_V86_MODE(pIemCpu))
3347	{
3348	pCtx->fpu.DS = 0;
3349	pCtx->fpu.FPUDP = (uint32_t)GCPtrEff \| ((uint32_t)sel << 4);
3350	}
3351	else
3352	{
3353	pCtx->fpu.DS = sel;
3354	pCtx->fpu.FPUDP = GCPtrEff;
3355	}
3356	}
3357
3358
3359	/**
3360	* Rotates the stack registers in the push direction.
3361	*
3362	* @param pCtx The CPU context.
3363	* @remarks This is a complete waste of time, but fxsave stores the registers in
3364	* stack order.
3365	*/
3366	DECLINLINE(void) iemFpuRotateStackPush(PCPUMCTX pCtx)
3367	{
3368	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[7].r80;
3369	pCtx->fpu.aRegs[7].r80 = pCtx->fpu.aRegs[6].r80;
3370	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[5].r80;
3371	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[4].r80;
3372	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[3].r80;
3373	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[2].r80;
3374	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[1].r80;
3375	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[0].r80;
3376	pCtx->fpu.aRegs[0].r80 = r80Tmp;
3377	}
3378
3379
3380	/**
3381	* Rotates the stack registers in the pop direction.
3382	*
3383	* @param pCtx The CPU context.
3384	* @remarks This is a complete waste of time, but fxsave stores the registers in
3385	* stack order.
3386	*/
3387	DECLINLINE(void) iemFpuRotateStackPop(PCPUMCTX pCtx)
3388	{
3389	RTFLOAT80U r80Tmp = pCtx->fpu.aRegs[0].r80;
3390	pCtx->fpu.aRegs[0].r80 = pCtx->fpu.aRegs[1].r80;
3391	pCtx->fpu.aRegs[1].r80 = pCtx->fpu.aRegs[2].r80;
3392	pCtx->fpu.aRegs[2].r80 = pCtx->fpu.aRegs[3].r80;
3393	pCtx->fpu.aRegs[3].r80 = pCtx->fpu.aRegs[4].r80;
3394	pCtx->fpu.aRegs[4].r80 = pCtx->fpu.aRegs[5].r80;
3395	pCtx->fpu.aRegs[5].r80 = pCtx->fpu.aRegs[6].r80;
3396	pCtx->fpu.aRegs[6].r80 = pCtx->fpu.aRegs[7].r80;
3397	pCtx->fpu.aRegs[7].r80 = r80Tmp;
3398	}
3399
3400
3401	/**
3402	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
3403	* exception prevents it.
3404	*
3405	* @param pIemCpu The IEM per CPU data.
3406	* @param pResult The FPU operation result to push.
3407	* @param pCtx The CPU context.
3408	*/
3409	static void iemFpuMaybePushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, PCPUMCTX pCtx)
3410	{
3411	/* Update FSW and bail if there are pending exceptions afterwards. */
3412	uint16_t fFsw = pCtx->fpu.FSW & ~X86_FSW_C_MASK;
3413	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3414	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3415	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3416	{
3417	pCtx->fpu.FSW = fFsw;
3418	return;
3419	}
3420
3421	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
3422	if (!(pCtx->fpu.FTW & RT_BIT(iNewTop)))
3423	{
3424	/* All is fine, push the actual value. */
3425	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3426	pCtx->fpu.aRegs[7].r80 = pResult->r80Result;
3427	}
3428	else if (pCtx->fpu.FCW & X86_FCW_IM)
3429	{
3430	/* Masked stack overflow, push QNaN. */
3431	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
3432	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3433	}
3434	else
3435	{
3436	/* Raise stack overflow, don't push anything. */
3437	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
3438	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
3439	return;
3440	}
3441
3442	fFsw &= ~X86_FSW_TOP_MASK;
3443	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
3444	pCtx->fpu.FSW = fFsw;
3445
3446	iemFpuRotateStackPush(pCtx);
3447	}
3448
3449
3450	/**
3451	* Stores a result in a FPU register and updates the FSW and FTW.
3452	*
3453	* @param pIemCpu The IEM per CPU data.
3454	* @param pResult The result to store.
3455	* @param iStReg Which FPU register to store it in.
3456	* @param pCtx The CPU context.
3457	*/
3458	static void iemFpuStoreResultOnly(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, PCPUMCTX pCtx)
3459	{
3460	Assert(iStReg < 8);
3461	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3462	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3463	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3464	pCtx->fpu.FTW \|= RT_BIT(iReg);
3465	pCtx->fpu.aRegs[iStReg].r80 = pResult->r80Result;
3466	}
3467
3468
3469	/**
3470	* Only updates the FPU status word (FSW) with the result of the current
3471	* instruction.
3472	*
3473	* @param pCtx The CPU context.
3474	* @param u16FSW The FSW output of the current instruction.
3475	*/
3476	static void iemFpuUpdateFSWOnly(PCPUMCTX pCtx, uint16_t u16FSW)
3477	{
3478	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3479	pCtx->fpu.FSW \|= u16FSW & ~X86_FSW_TOP_MASK;
3480	}
3481
3482
3483	/**
3484	* Pops one item off the FPU stack if no pending exception prevents it.
3485	*
3486	* @param pCtx The CPU context.
3487	*/
3488	static void iemFpuMaybePopOne(PCPUMCTX pCtx)
3489	{
3490	/* Check pending exceptions. */
3491	uint16_t uFSW = pCtx->fpu.FSW;
3492	if ( (pCtx->fpu.FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3493	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3494	return;
3495
3496	/* TOP--. */
3497	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
3498	uFSW &= ~X86_FSW_TOP_MASK;
3499	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3500	pCtx->fpu.FSW = uFSW;
3501
3502	/* Mark the previous ST0 as empty. */
3503	iOldTop >>= X86_FSW_TOP_SHIFT;
3504	pCtx->fpu.FTW &= ~RT_BIT(iOldTop);
3505
3506	/* Rotate the registers. */
3507	iemFpuRotateStackPop(pCtx);
3508	}
3509
3510
3511	/**
3512	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
3513	*
3514	* @param pIemCpu The IEM per CPU data.
3515	* @param pResult The FPU operation result to push.
3516	*/
3517	static void iemFpuPushResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult)
3518	{
3519	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3520	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3521	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
3522	}
3523
3524
3525	/**
3526	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
3527	* and sets FPUDP and FPUDS.
3528	*
3529	* @param pIemCpu The IEM per CPU data.
3530	* @param pResult The FPU operation result to push.
3531	* @param iEffSeg The effective segment register.
3532	* @param GCPtrEff The effective address relative to @a iEffSeg.
3533	*/
3534	static void iemFpuPushResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3535	{
3536	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3537	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3538	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3539	iemFpuMaybePushResult(pIemCpu, pResult, pCtx);
3540	}
3541
3542
3543	/**
3544	* Replace ST0 with the first value and push the second onto the FPU stack,
3545	* unless a pending exception prevents it.
3546	*
3547	* @param pIemCpu The IEM per CPU data.
3548	* @param pResult The FPU operation result to store and push.
3549	*/
3550	static void iemFpuPushResultTwo(PIEMCPU pIemCpu, PIEMFPURESULTTWO pResult)
3551	{
3552	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3553	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3554
3555	/* Update FSW and bail if there are pending exceptions afterwards. */
3556	uint16_t fFsw = pCtx->fpu.FSW & ~X86_FSW_C_MASK;
3557	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
3558	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
3559	& ~(pCtx->fpu.FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
3560	{
3561	pCtx->fpu.FSW = fFsw;
3562	return;
3563	}
3564
3565	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
3566	if (!(pCtx->fpu.FTW & RT_BIT(iNewTop)))
3567	{
3568	/* All is fine, push the actual value. */
3569	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3570	pCtx->fpu.aRegs[0].r80 = pResult->r80Result1;
3571	pCtx->fpu.aRegs[7].r80 = pResult->r80Result2;
3572	}
3573	else if (pCtx->fpu.FCW & X86_FCW_IM)
3574	{
3575	/* Masked stack overflow, push QNaN. */
3576	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
3577	iemFpuStoreQNan(&pCtx->fpu.aRegs[0].r80);
3578	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3579	}
3580	else
3581	{
3582	/* Raise stack overflow, don't push anything. */
3583	pCtx->fpu.FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
3584	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
3585	return;
3586	}
3587
3588	fFsw &= ~X86_FSW_TOP_MASK;
3589	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
3590	pCtx->fpu.FSW = fFsw;
3591
3592	iemFpuRotateStackPush(pCtx);
3593	}
3594
3595
3596	/**
3597	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
3598	* FOP.
3599	*
3600	* @param pIemCpu The IEM per CPU data.
3601	* @param pResult The result to store.
3602	* @param iStReg Which FPU register to store it in.
3603	* @param pCtx The CPU context.
3604	*/
3605	static void iemFpuStoreResult(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
3606	{
3607	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3608	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3609	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3610	}
3611
3612
3613	/**
3614	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
3615	* FOP, and then pops the stack.
3616	*
3617	* @param pIemCpu The IEM per CPU data.
3618	* @param pResult The result to store.
3619	* @param iStReg Which FPU register to store it in.
3620	* @param pCtx The CPU context.
3621	*/
3622	static void iemFpuStoreResultThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg)
3623	{
3624	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3625	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3626	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3627	iemFpuMaybePopOne(pCtx);
3628	}
3629
3630
3631	/**
3632	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
3633	* FPUDP, and FPUDS.
3634	*
3635	* @param pIemCpu The IEM per CPU data.
3636	* @param pResult The result to store.
3637	* @param iStReg Which FPU register to store it in.
3638	* @param pCtx The CPU context.
3639	* @param iEffSeg The effective memory operand selector register.
3640	* @param GCPtrEff The effective memory operand offset.
3641	*/
3642	static void iemFpuStoreResultWithMemOp(PIEMCPU pIemCpu, PIEMFPURESULT pResult, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3643	{
3644	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3645	iemFpuUpdateDP(pIemCpu, pIemCpu->CTX_SUFF(pCtx), iEffSeg, GCPtrEff);
3646	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3647	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3648	}
3649
3650
3651	/**
3652	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
3653	* FPUDP, and FPUDS, and then pops the stack.
3654	*
3655	* @param pIemCpu The IEM per CPU data.
3656	* @param pResult The result to store.
3657	* @param iStReg Which FPU register to store it in.
3658	* @param pCtx The CPU context.
3659	* @param iEffSeg The effective memory operand selector register.
3660	* @param GCPtrEff The effective memory operand offset.
3661	*/
3662	static void iemFpuStoreResultWithMemOpThenPop(PIEMCPU pIemCpu, PIEMFPURESULT pResult,
3663	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3664	{
3665	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3666	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3667	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3668	iemFpuStoreResultOnly(pIemCpu, pResult, iStReg, pCtx);
3669	iemFpuMaybePopOne(pCtx);
3670	}
3671
3672
3673	/**
3674	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
3675	*
3676	* @param pIemCpu The IEM per CPU data.
3677	*/
3678	static void iemFpuUpdateOpcodeAndIp(PIEMCPU pIemCpu)
3679	{
3680	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pIemCpu->CTX_SUFF(pCtx));
3681	}
3682
3683
3684	/**
3685	* Marks the specified stack register as free (for FFREE).
3686	*
3687	* @param pIemCpu The IEM per CPU data.
3688	* @param iStReg The register to free.
3689	*/
3690	static void iemFpuStackFree(PIEMCPU pIemCpu, uint8_t iStReg)
3691	{
3692	Assert(iStReg < 8);
3693	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3694	uint8_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3695	pCtx->fpu.FTW &= ~RT_BIT(iReg);
3696	}
3697
3698
3699	/**
3700	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
3701	*
3702	* @param pIemCpu The IEM per CPU data.
3703	*/
3704	static void iemFpuStackIncTop(PIEMCPU pIemCpu)
3705	{
3706	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3707	uint16_t uFsw = pCtx->fpu.FSW;
3708	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
3709	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3710	uFsw &= ~X86_FSW_TOP_MASK;
3711	uFsw \|= uTop;
3712	pCtx->fpu.FSW = uFsw;
3713	}
3714
3715
3716	/**
3717	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
3718	*
3719	* @param pIemCpu The IEM per CPU data.
3720	*/
3721	static void iemFpuStackDecTop(PIEMCPU pIemCpu)
3722	{
3723	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3724	uint16_t uFsw = pCtx->fpu.FSW;
3725	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
3726	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3727	uFsw &= ~X86_FSW_TOP_MASK;
3728	uFsw \|= uTop;
3729	pCtx->fpu.FSW = uFsw;
3730	}
3731
3732
3733	/**
3734	* Updates the FSW, FOP, FPUIP, and FPUCS.
3735	*
3736	* @param pIemCpu The IEM per CPU data.
3737	* @param u16FSW The FSW from the current instruction.
3738	*/
3739	static void iemFpuUpdateFSW(PIEMCPU pIemCpu, uint16_t u16FSW)
3740	{
3741	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3742	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3743	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3744	}
3745
3746
3747	/**
3748	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
3749	*
3750	* @param pIemCpu The IEM per CPU data.
3751	* @param u16FSW The FSW from the current instruction.
3752	*/
3753	static void iemFpuUpdateFSWThenPop(PIEMCPU pIemCpu, uint16_t u16FSW)
3754	{
3755	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3756	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3757	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3758	iemFpuMaybePopOne(pCtx);
3759	}
3760
3761
3762	/**
3763	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
3764	*
3765	* @param pIemCpu The IEM per CPU data.
3766	* @param u16FSW The FSW from the current instruction.
3767	* @param iEffSeg The effective memory operand selector register.
3768	* @param GCPtrEff The effective memory operand offset.
3769	*/
3770	static void iemFpuUpdateFSWWithMemOp(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3771	{
3772	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3773	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3774	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3775	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3776	}
3777
3778
3779	/**
3780	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
3781	*
3782	* @param pIemCpu The IEM per CPU data.
3783	* @param u16FSW The FSW from the current instruction.
3784	*/
3785	static void iemFpuUpdateFSWThenPopPop(PIEMCPU pIemCpu, uint16_t u16FSW)
3786	{
3787	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3788	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3789	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3790	iemFpuMaybePopOne(pCtx);
3791	iemFpuMaybePopOne(pCtx);
3792	}
3793
3794
3795	/**
3796	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
3797	*
3798	* @param pIemCpu The IEM per CPU data.
3799	* @param u16FSW The FSW from the current instruction.
3800	* @param iEffSeg The effective memory operand selector register.
3801	* @param GCPtrEff The effective memory operand offset.
3802	*/
3803	static void iemFpuUpdateFSWWithMemOpThenPop(PIEMCPU pIemCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3804	{
3805	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3806	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3807	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3808	iemFpuUpdateFSWOnly(pCtx, u16FSW);
3809	iemFpuMaybePopOne(pCtx);
3810	}
3811
3812
3813	/**
3814	* Worker routine for raising an FPU stack underflow exception.
3815	*
3816	* @param pIemCpu The IEM per CPU data.
3817	* @param iStReg The stack register being accessed.
3818	* @param pCtx The CPU context.
3819	*/
3820	static void iemFpuStackUnderflowOnly(PIEMCPU pIemCpu, uint8_t iStReg, PCPUMCTX pCtx)
3821	{
3822	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
3823	if (pCtx->fpu.FCW & X86_FCW_IM)
3824	{
3825	/* Masked underflow. */
3826	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3827	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
3828	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
3829	if (iStReg != UINT8_MAX)
3830	{
3831	pCtx->fpu.FTW \|= RT_BIT(iReg);
3832	iemFpuStoreQNan(&pCtx->fpu.aRegs[iStReg].r80);
3833	}
3834	}
3835	else
3836	{
3837	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3838	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3839	}
3840	}
3841
3842
3843	/**
3844	* Raises a FPU stack underflow exception.
3845	*
3846	* @param pIemCpu The IEM per CPU data.
3847	* @param iStReg The destination register that should be loaded
3848	* with QNaN if \#IS is not masked. Specify
3849	* UINT8_MAX if none (like for fcom).
3850	*/
3851	DECL_NO_INLINE(static, void) iemFpuStackUnderflow(PIEMCPU pIemCpu, uint8_t iStReg)
3852	{
3853	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3854	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3855	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3856	}
3857
3858
3859	DECL_NO_INLINE(static, void)
3860	iemFpuStackUnderflowWithMemOp(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3861	{
3862	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3863	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3864	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3865	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3866	}
3867
3868
3869	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPop(PIEMCPU pIemCpu, uint8_t iStReg)
3870	{
3871	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3872	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3873	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3874	iemFpuMaybePopOne(pCtx);
3875	}
3876
3877
3878	DECL_NO_INLINE(static, void)
3879	iemFpuStackUnderflowWithMemOpThenPop(PIEMCPU pIemCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
3880	{
3881	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3882	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
3883	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3884	iemFpuStackUnderflowOnly(pIemCpu, iStReg, pCtx);
3885	iemFpuMaybePopOne(pCtx);
3886	}
3887
3888
3889	DECL_NO_INLINE(static, void) iemFpuStackUnderflowThenPopPop(PIEMCPU pIemCpu)
3890	{
3891	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3892	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3893	iemFpuStackUnderflowOnly(pIemCpu, UINT8_MAX, pCtx);
3894	iemFpuMaybePopOne(pCtx);
3895	iemFpuMaybePopOne(pCtx);
3896	}
3897
3898
3899	DECL_NO_INLINE(static, void)
3900	iemFpuStackPushUnderflow(PIEMCPU pIemCpu)
3901	{
3902	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3903	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3904
3905	if (pCtx->fpu.FCW & X86_FCW_IM)
3906	{
3907	/* Masked overflow - Push QNaN. */
3908	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3909	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3910	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
3911	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3912	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3913	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3914	iemFpuRotateStackPush(pCtx);
3915	}
3916	else
3917	{
3918	/* Exception pending - don't change TOP or the register stack. */
3919	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3920	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3921	}
3922	}
3923
3924
3925	DECL_NO_INLINE(static, void)
3926	iemFpuStackPushUnderflowTwo(PIEMCPU pIemCpu)
3927	{
3928	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3929	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3930
3931	if (pCtx->fpu.FCW & X86_FCW_IM)
3932	{
3933	/* Masked overflow - Push QNaN. */
3934	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3935	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3936	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF;
3937	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3938	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3939	iemFpuStoreQNan(&pCtx->fpu.aRegs[0].r80);
3940	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3941	iemFpuRotateStackPush(pCtx);
3942	}
3943	else
3944	{
3945	/* Exception pending - don't change TOP or the register stack. */
3946	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3947	pCtx->fpu.FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3948	}
3949	}
3950
3951
3952	/**
3953	* Worker routine for raising an FPU stack overflow exception on a push.
3954	*
3955	* @param pIemCpu The IEM per CPU data.
3956	* @param pCtx The CPU context.
3957	*/
3958	static void iemFpuStackPushOverflowOnly(PIEMCPU pIemCpu, PCPUMCTX pCtx)
3959	{
3960	if (pCtx->fpu.FCW & X86_FCW_IM)
3961	{
3962	/* Masked overflow. */
3963	uint16_t iNewTop = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + 7) & X86_FSW_TOP_SMASK;
3964	pCtx->fpu.FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
3965	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
3966	pCtx->fpu.FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
3967	pCtx->fpu.FTW \|= RT_BIT(iNewTop);
3968	iemFpuStoreQNan(&pCtx->fpu.aRegs[7].r80);
3969	iemFpuRotateStackPush(pCtx);
3970	}
3971	else
3972	{
3973	/* Exception pending - don't change TOP or the register stack. */
3974	pCtx->fpu.FSW &= ~X86_FSW_C_MASK;
3975	pCtx->fpu.FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
3976	}
3977	}
3978
3979
3980	/**
3981	* Raises a FPU stack overflow exception on a push.
3982	*
3983	* @param pIemCpu The IEM per CPU data.
3984	*/
3985	DECL_NO_INLINE(static, void) iemFpuStackPushOverflow(PIEMCPU pIemCpu)
3986	{
3987	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
3988	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
3989	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
3990	}
3991
3992
3993	/**
3994	* Raises a FPU stack overflow exception on a push with a memory operand.
3995	*
3996	* @param pIemCpu The IEM per CPU data.
3997	* @param iEffSeg The effective memory operand selector register.
3998	* @param GCPtrEff The effective memory operand offset.
3999	*/
4000	DECL_NO_INLINE(static, void)
4001	iemFpuStackPushOverflowWithMemOp(PIEMCPU pIemCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
4002	{
4003	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4004	iemFpuUpdateDP(pIemCpu, pCtx, iEffSeg, GCPtrEff);
4005	iemFpuUpdateOpcodeAndIpWorker(pIemCpu, pCtx);
4006	iemFpuStackPushOverflowOnly(pIemCpu, pCtx);
4007	}
4008
4009
4010	static int iemFpuStRegNotEmpty(PIEMCPU pIemCpu, uint8_t iStReg)
4011	{
4012	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4013	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
4014	if (pCtx->fpu.FTW & RT_BIT(iReg))
4015	return VINF_SUCCESS;
4016	return VERR_NOT_FOUND;
4017	}
4018
4019
4020	static int iemFpuStRegNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
4021	{
4022	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4023	uint16_t iReg = (X86_FSW_TOP_GET(pCtx->fpu.FSW) + iStReg) & X86_FSW_TOP_SMASK;
4024	if (pCtx->fpu.FTW & RT_BIT(iReg))
4025	{
4026	*ppRef = &pCtx->fpu.aRegs[iStReg].r80;
4027	return VINF_SUCCESS;
4028	}
4029	return VERR_NOT_FOUND;
4030	}
4031
4032
4033	static int iemFpu2StRegsNotEmptyRef(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
4034	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
4035	{
4036	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4037	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
4038	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
4039	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
4040	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
4041	{
4042	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
4043	*ppRef1 = &pCtx->fpu.aRegs[iStReg1].r80;
4044	return VINF_SUCCESS;
4045	}
4046	return VERR_NOT_FOUND;
4047	}
4048
4049
4050	static int iemFpu2StRegsNotEmptyRefFirst(PIEMCPU pIemCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
4051	{
4052	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
4053	uint16_t iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
4054	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
4055	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
4056	if ((pCtx->fpu.FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
4057	{
4058	*ppRef0 = &pCtx->fpu.aRegs[iStReg0].r80;
4059	return VINF_SUCCESS;
4060	}
4061	return VERR_NOT_FOUND;
4062	}
4063
4064
4065	/**
4066	* Updates the FPU exception status after FCW is changed.
4067	*
4068	* @param pCtx The CPU context.
4069	*/
4070	static void iemFpuRecalcExceptionStatus(PCPUMCTX pCtx)
4071	{
4072	uint16_t u16Fsw = pCtx->fpu.FSW;
4073	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pCtx->fpu.FCW & X86_FCW_XCPT_MASK))
4074	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
4075	else
4076	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
4077	pCtx->fpu.FSW = u16Fsw;
4078	}
4079
4080
4081	/**
4082	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
4083	*
4084	* @returns The full FTW.
4085	* @param pCtx The CPU state.
4086	*/
4087	static uint16_t iemFpuCalcFullFtw(PCCPUMCTX pCtx)
4088	{
4089	uint8_t const u8Ftw = (uint8_t)pCtx->fpu.FTW;
4090	uint16_t u16Ftw = 0;
4091	unsigned const iTop = X86_FSW_TOP_GET(pCtx->fpu.FSW);
4092	for (unsigned iSt = 0; iSt < 8; iSt++)
4093	{
4094	unsigned const iReg = (iSt + iTop) & 7;
4095	if (!(u8Ftw & RT_BIT(iReg)))
4096	u16Ftw \|= 3 << (iReg * 2); /* empty */
4097	else
4098	{
4099	uint16_t uTag;
4100	PCRTFLOAT80U const pr80Reg = &pCtx->fpu.aRegs[iSt].r80;
4101	if (pr80Reg->s.uExponent == 0x7fff)
4102	uTag = 2; /* Exponent is all 1's => Special. */
4103	else if (pr80Reg->s.uExponent == 0x0000)
4104	{
4105	if (pr80Reg->s.u64Mantissa == 0x0000)
4106	uTag = 1; /* All bits are zero => Zero. */
4107	else
4108	uTag = 2; /* Must be special. */
4109	}
4110	else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
4111	uTag = 0; /* Valid. */
4112	else
4113	uTag = 2; /* Must be special. */
4114
4115	u16Ftw \|= uTag << (iReg * 2); /* empty */
4116	}
4117	}
4118
4119	return u16Ftw;
4120	}
4121
4122
4123	/**
4124	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
4125	*
4126	* @returns The compressed FTW.
4127	* @param u16FullFtw The full FTW to convert.
4128	*/
4129	static uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
4130	{
4131	uint8_t u8Ftw = 0;
4132	for (unsigned i = 0; i < 8; i++)
4133	{
4134	if ((u16FullFtw & 3) != 3 /empty/)
4135	u8Ftw \|= RT_BIT(i);
4136	u16FullFtw >>= 2;
4137	}
4138
4139	return u8Ftw;
4140	}
4141
4142	/** @} */
4143
4144
4145	/** @name Memory access.
4146	*
4147	* @{
4148	*/
4149
4150
4151	/**
4152	* Checks if the given segment can be written to, raise the appropriate
4153	* exception if not.
4154	*
4155	* @returns VBox strict status code.
4156	*
4157	* @param pIemCpu The IEM per CPU data.
4158	* @param pHid Pointer to the hidden register.
4159	* @param iSegReg The register number.
4160	*/
4161	static VBOXSTRICTRC iemMemSegCheckWriteAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg)
4162	{
4163	if (!pHid->Attr.n.u1Present)
4164	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
4165
4166	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
4167	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
4168	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
4169	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_W);
4170
4171	/** @todo DPL/RPL/CPL? */
4172
4173	return VINF_SUCCESS;
4174	}
4175
4176
4177	/**
4178	* Checks if the given segment can be read from, raise the appropriate
4179	* exception if not.
4180	*
4181	* @returns VBox strict status code.
4182	*
4183	* @param pIemCpu The IEM per CPU data.
4184	* @param pHid Pointer to the hidden register.
4185	* @param iSegReg The register number.
4186	*/
4187	static VBOXSTRICTRC iemMemSegCheckReadAccessEx(PIEMCPU pIemCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg)
4188	{
4189	if (!pHid->Attr.n.u1Present)
4190	return iemRaiseSelectorNotPresentBySegReg(pIemCpu, iSegReg);
4191
4192	if ( (pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE
4193	&& pIemCpu->enmCpuMode != IEMMODE_64BIT )
4194	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, IEM_ACCESS_DATA_R);
4195
4196	/** @todo DPL/RPL/CPL? */
4197
4198	return VINF_SUCCESS;
4199	}
4200
4201
4202	/**
4203	* Applies the segment limit, base and attributes.
4204	*
4205	* This may raise a \#GP or \#SS.
4206	*
4207	* @returns VBox strict status code.
4208	*
4209	* @param pIemCpu The IEM per CPU data.
4210	* @param fAccess The kind of access which is being performed.
4211	* @param iSegReg The index of the segment register to apply.
4212	* This is UINT8_MAX if none (for IDT, GDT, LDT,
4213	* TSS, ++).
4214	* @param pGCPtrMem Pointer to the guest memory address to apply
4215	* segmentation to. Input and output parameter.
4216	*/
4217	static VBOXSTRICTRC iemMemApplySegment(PIEMCPU pIemCpu, uint32_t fAccess, uint8_t iSegReg,
4218	size_t cbMem, PRTGCPTR pGCPtrMem)
4219	{
4220	if (iSegReg == UINT8_MAX)
4221	return VINF_SUCCESS;
4222
4223	PCPUMSELREGHID pSel = iemSRegGetHid(pIemCpu, iSegReg);
4224	switch (pIemCpu->enmCpuMode)
4225	{
4226	case IEMMODE_16BIT:
4227	case IEMMODE_32BIT:
4228	{
4229	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
4230	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
4231
4232	Assert(pSel->Attr.n.u1Present);
4233	Assert(pSel->Attr.n.u1DescType);
4234	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
4235	{
4236	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4237	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
4238	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
4239
4240	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
4241	{
4242	/** @todo CPL check. */
4243	}
4244
4245	/*
4246	* There are two kinds of data selectors, normal and expand down.
4247	*/
4248	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
4249	{
4250	if ( GCPtrFirst32 > pSel->u32Limit
4251	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
4252	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
4253
4254	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
4255	}
4256	else
4257	{
4258	/** @todo implement expand down segments. */
4259	AssertFailed(/** @todo implement this */);
4260	return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
4261	}
4262	}
4263	else
4264	{
4265
4266	/*
4267	* Code selector and usually be used to read thru, writing is
4268	* only permitted in real and V8086 mode.
4269	*/
4270	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4271	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
4272	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
4273	&& !IEM_IS_REAL_OR_V86_MODE(pIemCpu) )
4274	return iemRaiseSelectorInvalidAccess(pIemCpu, iSegReg, fAccess);
4275
4276	if ( GCPtrFirst32 > pSel->u32Limit
4277	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
4278	return iemRaiseSelectorBounds(pIemCpu, iSegReg, fAccess);
4279
4280	if (!IEM_IS_REAL_OR_V86_MODE(pIemCpu))
4281	{
4282	/** @todo CPL check. */
4283	}
4284
4285	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
4286	}
4287	return VINF_SUCCESS;
4288	}
4289
4290	case IEMMODE_64BIT:
4291	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
4292	*pGCPtrMem += pSel->u64Base;
4293	return VINF_SUCCESS;
4294
4295	default:
4296	AssertFailedReturn(VERR_INTERNAL_ERROR_5);
4297	}
4298	}
4299
4300
4301	/**
4302	* Translates a virtual address to a physical physical address and checks if we
4303	* can access the page as specified.
4304	*
4305	* @param pIemCpu The IEM per CPU data.
4306	* @param GCPtrMem The virtual address.
4307	* @param fAccess The intended access.
4308	* @param pGCPhysMem Where to return the physical address.
4309	*/
4310	static VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PIEMCPU pIemCpu, RTGCPTR GCPtrMem, uint32_t fAccess,
4311	PRTGCPHYS pGCPhysMem)
4312	{
4313	/** @todo Need a different PGM interface here. We're currently using
4314	* generic / REM interfaces. this won't cut it for R0 & RC. */
4315	RTGCPHYS GCPhys;
4316	uint64_t fFlags;
4317	int rc = PGMGstGetPage(IEMCPU_TO_VMCPU(pIemCpu), GCPtrMem, &fFlags, &GCPhys);
4318	if (RT_FAILURE(rc))
4319	{
4320	/** @todo Check unassigned memory in unpaged mode. */
4321	/** @todo Reserved bits in page tables. Requires new PGM interface. */
4322	*pGCPhysMem = NIL_RTGCPHYS;
4323	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, rc);
4324	}
4325
4326	/* If the page is writable and does not have the no-exec bit set, all
4327	access is allowed. Otherwise we'll have to check more carefully... */
4328	if ((fFlags & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
4329	{
4330	/* Write to read only memory? */
4331	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
4332	&& !(fFlags & X86_PTE_RW)
4333	&& ( pIemCpu->uCpl != 0
4334	\|\| (pIemCpu->CTX_SUFF(pCtx)->cr0 & X86_CR0_WP)))
4335	{
4336	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
4337	*pGCPhysMem = NIL_RTGCPHYS;
4338	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
4339	}
4340
4341	/* Kernel memory accessed by userland? */
4342	if ( !(fFlags & X86_PTE_US)
4343	&& pIemCpu->uCpl == 3
4344	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
4345	{
4346	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
4347	*pGCPhysMem = NIL_RTGCPHYS;
4348	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
4349	}
4350
4351	/* Executing non-executable memory? */
4352	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
4353	&& (fFlags & X86_PTE_PAE_NX)
4354	&& (pIemCpu->CTX_SUFF(pCtx)->msrEFER & MSR_K6_EFER_NXE) )
4355	{
4356	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
4357	*pGCPhysMem = NIL_RTGCPHYS;
4358	return iemRaisePageFault(pIemCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
4359	VERR_ACCESS_DENIED);
4360	}
4361	}
4362
4363	GCPhys \|= GCPtrMem & PAGE_OFFSET_MASK;
4364	*pGCPhysMem = GCPhys;
4365	return VINF_SUCCESS;
4366	}
4367
4368
4369
4370	/**
4371	* Maps a physical page.
4372	*
4373	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
4374	* @param pIemCpu The IEM per CPU data.
4375	* @param GCPhysMem The physical address.
4376	* @param fAccess The intended access.
4377	* @param ppvMem Where to return the mapping address.
4378	*/
4379	static int iemMemPageMap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem)
4380	{
4381	#ifdef IEM_VERIFICATION_MODE
4382	/* Force the alternative path so we can ignore writes. */
4383	if ((fAccess & IEM_ACCESS_TYPE_WRITE) && !pIemCpu->fNoRem)
4384	return VERR_PGM_PHYS_TLB_CATCH_ALL;
4385	#endif
4386
4387	/*
4388	* If we can map the page without trouble, do a block processing
4389	* until the end of the current page.
4390	*/
4391	/** @todo need some better API. */
4392	return PGMR3PhysTlbGCPhys2Ptr(IEMCPU_TO_VM(pIemCpu),
4393	GCPhysMem,
4394	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
4395	ppvMem);
4396	}
4397
4398
4399	/**
4400	* Unmap a page previously mapped by iemMemPageMap.
4401	*
4402	* This is currently a dummy function.
4403	*
4404	* @param pIemCpu The IEM per CPU data.
4405	* @param GCPhysMem The physical address.
4406	* @param fAccess The intended access.
4407	* @param pvMem What iemMemPageMap returned.
4408	*/
4409	DECLINLINE(void) iemMemPageUnmap(PIEMCPU pIemCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem)
4410	{
4411	NOREF(pIemCpu);
4412	NOREF(GCPhysMem);
4413	NOREF(fAccess);
4414	NOREF(pvMem);
4415	}
4416
4417
4418	/**
4419	* Looks up a memory mapping entry.
4420	*
4421	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
4422	* @param pIemCpu The IEM per CPU data.
4423	* @param pvMem The memory address.
4424	* @param fAccess The access to.
4425	*/
4426	DECLINLINE(int) iemMapLookup(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
4427	{
4428	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
4429	if ( pIemCpu->aMemMappings[0].pv == pvMem
4430	&& (pIemCpu->aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4431	return 0;
4432	if ( pIemCpu->aMemMappings[1].pv == pvMem
4433	&& (pIemCpu->aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4434	return 1;
4435	if ( pIemCpu->aMemMappings[2].pv == pvMem
4436	&& (pIemCpu->aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
4437	return 2;
4438	return VERR_NOT_FOUND;
4439	}
4440
4441
4442	/**
4443	* Finds a free memmap entry when using iNextMapping doesn't work.
4444	*
4445	* @returns Memory mapping index, 1024 on failure.
4446	* @param pIemCpu The IEM per CPU data.
4447	*/
4448	static unsigned iemMemMapFindFree(PIEMCPU pIemCpu)
4449	{
4450	/*
4451	* The easy case.
4452	*/
4453	if (pIemCpu->cActiveMappings == 0)
4454	{
4455	pIemCpu->iNextMapping = 1;
4456	return 0;
4457	}
4458
4459	/* There should be enough mappings for all instructions. */
4460	AssertReturn(pIemCpu->cActiveMappings < RT_ELEMENTS(pIemCpu->aMemMappings), 1024);
4461
4462	for (unsigned i = 0; i < RT_ELEMENTS(pIemCpu->aMemMappings); i++)
4463	if (pIemCpu->aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
4464	return i;
4465
4466	AssertFailedReturn(1024);
4467	}
4468
4469
4470	/**
4471	* Commits a bounce buffer that needs writing back and unmaps it.
4472	*
4473	* @returns Strict VBox status code.
4474	* @param pIemCpu The IEM per CPU data.
4475	* @param iMemMap The index of the buffer to commit.
4476	*/
4477	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PIEMCPU pIemCpu, unsigned iMemMap)
4478	{
4479	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
4480	Assert(pIemCpu->aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
4481
4482	/*
4483	* Do the writing.
4484	*/
4485	int rc;
4486	if ( !pIemCpu->aMemBbMappings[iMemMap].fUnassigned
4487	&& !IEM_VERIFICATION_ENABLED(pIemCpu))
4488	{
4489	uint16_t const cbFirst = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
4490	uint16_t const cbSecond = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
4491	uint8_t const *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4492	if (!pIemCpu->fByPassHandlers)
4493	{
4494	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
4495	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
4496	pbBuf,
4497	cbFirst);
4498	if (cbSecond && rc == VINF_SUCCESS)
4499	rc = PGMPhysWrite(IEMCPU_TO_VM(pIemCpu),
4500	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
4501	pbBuf + cbFirst,
4502	cbSecond);
4503	}
4504	else
4505	{
4506	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
4507	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst,
4508	pbBuf,
4509	cbFirst);
4510	if (cbSecond && rc == VINF_SUCCESS)
4511	rc = PGMPhysSimpleWriteGCPhys(IEMCPU_TO_VM(pIemCpu),
4512	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond,
4513	pbBuf + cbFirst,
4514	cbSecond);
4515	}
4516	}
4517	else
4518	rc = VINF_SUCCESS;
4519
4520	#ifdef IEM_VERIFICATION_MODE
4521	/*
4522	* Record the write(s).
4523	*/
4524	if (!pIemCpu->fNoRem)
4525	{
4526	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4527	if (pEvtRec)
4528	{
4529	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
4530	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst;
4531	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbFirst;
4532	memcpy(pEvtRec->u.RamWrite.ab, &pIemCpu->aBounceBuffers[iMemMap].ab[0], pIemCpu->aMemBbMappings[iMemMap].cbFirst);
4533	AssertCompile(sizeof(pEvtRec->u.RamWrite.ab) == sizeof(pIemCpu->aBounceBuffers[0].ab));
4534	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4535	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4536	}
4537	if (pIemCpu->aMemBbMappings[iMemMap].cbSecond)
4538	{
4539	pEvtRec = iemVerifyAllocRecord(pIemCpu);
4540	if (pEvtRec)
4541	{
4542	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
4543	pEvtRec->u.RamWrite.GCPhys = pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond;
4544	pEvtRec->u.RamWrite.cb = pIemCpu->aMemBbMappings[iMemMap].cbSecond;
4545	memcpy(pEvtRec->u.RamWrite.ab,
4546	&pIemCpu->aBounceBuffers[iMemMap].ab[pIemCpu->aMemBbMappings[iMemMap].cbFirst],
4547	pIemCpu->aMemBbMappings[iMemMap].cbSecond);
4548	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4549	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4550	}
4551	}
4552	}
4553	#endif
4554
4555	/*
4556	* Free the mapping entry.
4557	*/
4558	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
4559	Assert(pIemCpu->cActiveMappings != 0);
4560	pIemCpu->cActiveMappings--;
4561	return rc;
4562	}
4563
4564
4565	/**
4566	* iemMemMap worker that deals with a request crossing pages.
4567	*/
4568	static VBOXSTRICTRC iemMemBounceBufferMapCrossPage(PIEMCPU pIemCpu, int iMemMap, void **ppvMem,
4569	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
4570	{
4571	/*
4572	* Do the address translations.
4573	*/
4574	RTGCPHYS GCPhysFirst;
4575	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst, fAccess, &GCPhysFirst);
4576	if (rcStrict != VINF_SUCCESS)
4577	return rcStrict;
4578
4579	RTGCPHYS GCPhysSecond;
4580	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrFirst + (cbMem - 1), fAccess, &GCPhysSecond);
4581	if (rcStrict != VINF_SUCCESS)
4582	return rcStrict;
4583	GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
4584
4585	/*
4586	* Read in the current memory content if it's a read, execute or partial
4587	* write access.
4588	*/
4589	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4590	uint32_t const cbFirstPage = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
4591	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
4592
4593	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
4594	{
4595	int rc;
4596	if (!pIemCpu->fByPassHandlers)
4597	{
4598	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbFirstPage);
4599	if (rc != VINF_SUCCESS)
4600	return rc;
4601	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage);
4602	if (rc != VINF_SUCCESS)
4603	return rc;
4604	}
4605	else
4606	{
4607	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbFirstPage);
4608	if (rc != VINF_SUCCESS)
4609	return rc;
4610	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
4611	if (rc != VINF_SUCCESS)
4612	return rc;
4613	}
4614
4615	#ifdef IEM_VERIFICATION_MODE
4616	if ( !pIemCpu->fNoRem
4617	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
4618	{
4619	/*
4620	* Record the reads.
4621	*/
4622	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4623	if (pEvtRec)
4624	{
4625	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4626	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
4627	pEvtRec->u.RamRead.cb = cbFirstPage;
4628	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4629	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4630	}
4631	pEvtRec = iemVerifyAllocRecord(pIemCpu);
4632	if (pEvtRec)
4633	{
4634	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4635	pEvtRec->u.RamRead.GCPhys = GCPhysSecond;
4636	pEvtRec->u.RamRead.cb = cbSecondPage;
4637	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4638	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4639	}
4640	}
4641	#endif
4642	}
4643	#ifdef VBOX_STRICT
4644	else
4645	memset(pbBuf, 0xcc, cbMem);
4646	#endif
4647	#ifdef VBOX_STRICT
4648	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
4649	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
4650	#endif
4651
4652	/*
4653	* Commit the bounce buffer entry.
4654	*/
4655	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
4656	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
4657	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
4658	pIemCpu->aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
4659	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = false;
4660	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
4661	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
4662	pIemCpu->cActiveMappings++;
4663
4664	*ppvMem = pbBuf;
4665	return VINF_SUCCESS;
4666	}
4667
4668
4669	/**
4670	* iemMemMap woker that deals with iemMemPageMap failures.
4671	*/
4672	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PIEMCPU pIemCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
4673	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
4674	{
4675	/*
4676	* Filter out conditions we can handle and the ones which shouldn't happen.
4677	*/
4678	if ( rcMap != VINF_PGM_PHYS_TLB_CATCH_WRITE
4679	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
4680	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
4681	{
4682	AssertReturn(RT_FAILURE_NP(rcMap), VERR_INTERNAL_ERROR_3);
4683	return rcMap;
4684	}
4685	pIemCpu->cPotentialExits++;
4686
4687	/*
4688	* Read in the current memory content if it's a read, execute or partial
4689	* write access.
4690	*/
4691	uint8_t *pbBuf = &pIemCpu->aBounceBuffers[iMemMap].ab[0];
4692	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
4693	{
4694	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
4695	memset(pbBuf, 0xff, cbMem);
4696	else
4697	{
4698	int rc;
4699	if (!pIemCpu->fByPassHandlers)
4700	rc = PGMPhysRead(IEMCPU_TO_VM(pIemCpu), GCPhysFirst, pbBuf, cbMem);
4701	else
4702	rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), pbBuf, GCPhysFirst, cbMem);
4703	if (rc != VINF_SUCCESS)
4704	return rc;
4705	}
4706
4707	#ifdef IEM_VERIFICATION_MODE
4708	if ( !pIemCpu->fNoRem
4709	&& (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC)) )
4710	{
4711	/*
4712	* Record the read.
4713	*/
4714	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
4715	if (pEvtRec)
4716	{
4717	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
4718	pEvtRec->u.RamRead.GCPhys = GCPhysFirst;
4719	pEvtRec->u.RamRead.cb = (uint32_t)cbMem;
4720	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
4721	*pIemCpu->ppIemEvtRecNext = pEvtRec;
4722	}
4723	}
4724	#endif
4725	}
4726	#ifdef VBOX_STRICT
4727	else
4728	memset(pbBuf, 0xcc, cbMem);
4729	#endif
4730	#ifdef VBOX_STRICT
4731	if (cbMem < sizeof(pIemCpu->aBounceBuffers[iMemMap].ab))
4732	memset(pbBuf + cbMem, 0xaa, sizeof(pIemCpu->aBounceBuffers[iMemMap].ab) - cbMem);
4733	#endif
4734
4735	/*
4736	* Commit the bounce buffer entry.
4737	*/
4738	pIemCpu->aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
4739	pIemCpu->aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
4740	pIemCpu->aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
4741	pIemCpu->aMemBbMappings[iMemMap].cbSecond = 0;
4742	pIemCpu->aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
4743	pIemCpu->aMemMappings[iMemMap].pv = pbBuf;
4744	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
4745	pIemCpu->cActiveMappings++;
4746
4747	*ppvMem = pbBuf;
4748	return VINF_SUCCESS;
4749	}
4750
4751
4752
4753	/**
4754	* Maps the specified guest memory for the given kind of access.
4755	*
4756	* This may be using bounce buffering of the memory if it's crossing a page
4757	* boundary or if there is an access handler installed for any of it. Because
4758	* of lock prefix guarantees, we're in for some extra clutter when this
4759	* happens.
4760	*
4761	* This may raise a \#GP, \#SS, \#PF or \#AC.
4762	*
4763	* @returns VBox strict status code.
4764	*
4765	* @param pIemCpu The IEM per CPU data.
4766	* @param ppvMem Where to return the pointer to the mapped
4767	* memory.
4768	* @param cbMem The number of bytes to map. This is usually 1,
4769	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
4770	* string operations it can be up to a page.
4771	* @param iSegReg The index of the segment register to use for
4772	* this access. The base and limits are checked.
4773	* Use UINT8_MAX to indicate that no segmentation
4774	* is required (for IDT, GDT and LDT accesses).
4775	* @param GCPtrMem The address of the guest memory.
4776	* @param a_fAccess How the memory is being accessed. The
4777	* IEM_ACCESS_TYPE_XXX bit is used to figure out
4778	* how to map the memory, while the
4779	* IEM_ACCESS_WHAT_XXX bit is used when raising
4780	* exceptions.
4781	*/
4782	static VBOXSTRICTRC iemMemMap(PIEMCPU pIemCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
4783	{
4784	/*
4785	* Check the input and figure out which mapping entry to use.
4786	*/
4787	Assert(cbMem <= 32 \|\| cbMem == 512);
4788	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
4789
4790	unsigned iMemMap = pIemCpu->iNextMapping;
4791	if (iMemMap >= RT_ELEMENTS(pIemCpu->aMemMappings))
4792	{
4793	iMemMap = iemMemMapFindFree(pIemCpu);
4794	AssertReturn(iMemMap < RT_ELEMENTS(pIemCpu->aMemMappings), VERR_INTERNAL_ERROR_3);
4795	}
4796
4797	/*
4798	* Map the memory, checking that we can actually access it. If something
4799	* slightly complicated happens, fall back on bounce buffering.
4800	*/
4801	VBOXSTRICTRC rcStrict = iemMemApplySegment(pIemCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
4802	if (rcStrict != VINF_SUCCESS)
4803	return rcStrict;
4804
4805	if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
4806	return iemMemBounceBufferMapCrossPage(pIemCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
4807
4808	RTGCPHYS GCPhysFirst;
4809	rcStrict = iemMemPageTranslateAndCheckAccess(pIemCpu, GCPtrMem, fAccess, &GCPhysFirst);
4810	if (rcStrict != VINF_SUCCESS)
4811	return rcStrict;
4812
4813	void *pvMem;
4814	rcStrict = iemMemPageMap(pIemCpu, GCPhysFirst, fAccess, &pvMem);
4815	if (rcStrict != VINF_SUCCESS)
4816	return iemMemBounceBufferMapPhys(pIemCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
4817
4818	/*
4819	* Fill in the mapping table entry.
4820	*/
4821	pIemCpu->aMemMappings[iMemMap].pv = pvMem;
4822	pIemCpu->aMemMappings[iMemMap].fAccess = fAccess;
4823	pIemCpu->iNextMapping = iMemMap + 1;
4824	pIemCpu->cActiveMappings++;
4825
4826	*ppvMem = pvMem;
4827	return VINF_SUCCESS;
4828	}
4829
4830
4831	/**
4832	* Commits the guest memory if bounce buffered and unmaps it.
4833	*
4834	* @returns Strict VBox status code.
4835	* @param pIemCpu The IEM per CPU data.
4836	* @param pvMem The mapping.
4837	* @param fAccess The kind of access.
4838	*/
4839	static VBOXSTRICTRC iemMemCommitAndUnmap(PIEMCPU pIemCpu, void *pvMem, uint32_t fAccess)
4840	{
4841	int iMemMap = iemMapLookup(pIemCpu, pvMem, fAccess);
4842	AssertReturn(iMemMap >= 0, iMemMap);
4843
4844	/*
4845	* If it's bounce buffered, we need to write back the buffer.
4846	*/
4847	if ( (pIemCpu->aMemMappings[iMemMap].fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_TYPE_WRITE))
4848	== (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_TYPE_WRITE))
4849	return iemMemBounceBufferCommitAndUnmap(pIemCpu, iMemMap);
4850
4851	/* Free the entry. */
4852	pIemCpu->aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
4853	Assert(pIemCpu->cActiveMappings != 0);
4854	pIemCpu->cActiveMappings--;
4855	return VINF_SUCCESS;
4856	}
4857
4858
4859	/**
4860	* Fetches a data byte.
4861	*
4862	* @returns Strict VBox status code.
4863	* @param pIemCpu The IEM per CPU data.
4864	* @param pu8Dst Where to return the byte.
4865	* @param iSegReg The index of the segment register to use for
4866	* this access. The base and limits are checked.
4867	* @param GCPtrMem The address of the guest memory.
4868	*/
4869	static VBOXSTRICTRC iemMemFetchDataU8(PIEMCPU pIemCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4870	{
4871	/* The lazy approach for now... */
4872	uint8_t const *pu8Src;
4873	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4874	if (rc == VINF_SUCCESS)
4875	{
4876	pu8Dst = pu8Src;
4877	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
4878	}
4879	return rc;
4880	}
4881
4882
4883	/**
4884	* Fetches a data word.
4885	*
4886	* @returns Strict VBox status code.
4887	* @param pIemCpu The IEM per CPU data.
4888	* @param pu16Dst Where to return the word.
4889	* @param iSegReg The index of the segment register to use for
4890	* this access. The base and limits are checked.
4891	* @param GCPtrMem The address of the guest memory.
4892	*/
4893	static VBOXSTRICTRC iemMemFetchDataU16(PIEMCPU pIemCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4894	{
4895	/* The lazy approach for now... */
4896	uint16_t const *pu16Src;
4897	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4898	if (rc == VINF_SUCCESS)
4899	{
4900	pu16Dst = pu16Src;
4901	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
4902	}
4903	return rc;
4904	}
4905
4906
4907	/**
4908	* Fetches a data dword.
4909	*
4910	* @returns Strict VBox status code.
4911	* @param pIemCpu The IEM per CPU data.
4912	* @param pu32Dst Where to return the dword.
4913	* @param iSegReg The index of the segment register to use for
4914	* this access. The base and limits are checked.
4915	* @param GCPtrMem The address of the guest memory.
4916	*/
4917	static VBOXSTRICTRC iemMemFetchDataU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4918	{
4919	/* The lazy approach for now... */
4920	uint32_t const *pu32Src;
4921	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4922	if (rc == VINF_SUCCESS)
4923	{
4924	pu32Dst = pu32Src;
4925	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
4926	}
4927	return rc;
4928	}
4929
4930
4931	#ifdef SOME_UNUSED_FUNCTION
4932	/**
4933	* Fetches a data dword and sign extends it to a qword.
4934	*
4935	* @returns Strict VBox status code.
4936	* @param pIemCpu The IEM per CPU data.
4937	* @param pu64Dst Where to return the sign extended value.
4938	* @param iSegReg The index of the segment register to use for
4939	* this access. The base and limits are checked.
4940	* @param GCPtrMem The address of the guest memory.
4941	*/
4942	static VBOXSTRICTRC iemMemFetchDataS32SxU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4943	{
4944	/* The lazy approach for now... */
4945	int32_t const *pi32Src;
4946	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4947	if (rc == VINF_SUCCESS)
4948	{
4949	pu64Dst = pi32Src;
4950	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
4951	}
4952	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
4953	else
4954	*pu64Dst = 0;
4955	#endif
4956	return rc;
4957	}
4958	#endif
4959
4960
4961	/**
4962	* Fetches a data qword.
4963	*
4964	* @returns Strict VBox status code.
4965	* @param pIemCpu The IEM per CPU data.
4966	* @param pu64Dst Where to return the qword.
4967	* @param iSegReg The index of the segment register to use for
4968	* this access. The base and limits are checked.
4969	* @param GCPtrMem The address of the guest memory.
4970	*/
4971	static VBOXSTRICTRC iemMemFetchDataU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4972	{
4973	/* The lazy approach for now... */
4974	uint64_t const *pu64Src;
4975	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
4976	if (rc == VINF_SUCCESS)
4977	{
4978	pu64Dst = pu64Src;
4979	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
4980	}
4981	return rc;
4982	}
4983
4984
4985	/**
4986	* Fetches a data tword.
4987	*
4988	* @returns Strict VBox status code.
4989	* @param pIemCpu The IEM per CPU data.
4990	* @param pr80Dst Where to return the tword.
4991	* @param iSegReg The index of the segment register to use for
4992	* this access. The base and limits are checked.
4993	* @param GCPtrMem The address of the guest memory.
4994	*/
4995	static VBOXSTRICTRC iemMemFetchDataR80(PIEMCPU pIemCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
4996	{
4997	/* The lazy approach for now... */
4998	PCRTFLOAT80U pr80Src;
4999	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pr80Src, sizeof(pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
5000	if (rc == VINF_SUCCESS)
5001	{
5002	pr80Dst = pr80Src;
5003	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
5004	}
5005	return rc;
5006	}
5007
5008
5009	/**
5010	* Fetches a descriptor register (lgdt, lidt).
5011	*
5012	* @returns Strict VBox status code.
5013	* @param pIemCpu The IEM per CPU data.
5014	* @param pcbLimit Where to return the limit.
5015	* @param pGCPTrBase Where to return the base.
5016	* @param iSegReg The index of the segment register to use for
5017	* this access. The base and limits are checked.
5018	* @param GCPtrMem The address of the guest memory.
5019	* @param enmOpSize The effective operand size.
5020	*/
5021	static VBOXSTRICTRC iemMemFetchDataXdtr(PIEMCPU pIemCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase,
5022	uint8_t iSegReg, RTGCPTR GCPtrMem, IEMMODE enmOpSize)
5023	{
5024	uint8_t const *pu8Src;
5025	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu,
5026	(void **)&pu8Src,
5027	enmOpSize == IEMMODE_64BIT
5028	? 2 + 8
5029	: enmOpSize == IEMMODE_32BIT
5030	? 2 + 4
5031	: 2 + 3,
5032	iSegReg,
5033	GCPtrMem,
5034	IEM_ACCESS_DATA_R);
5035	if (rcStrict == VINF_SUCCESS)
5036	{
5037	*pcbLimit = RT_MAKE_U16(pu8Src[0], pu8Src[1]);
5038	switch (enmOpSize)
5039	{
5040	case IEMMODE_16BIT:
5041	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], 0);
5042	break;
5043	case IEMMODE_32BIT:
5044	*pGCPtrBase = RT_MAKE_U32_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5]);
5045	break;
5046	case IEMMODE_64BIT:
5047	*pGCPtrBase = RT_MAKE_U64_FROM_U8(pu8Src[2], pu8Src[3], pu8Src[4], pu8Src[5],
5048	pu8Src[6], pu8Src[7], pu8Src[8], pu8Src[9]);
5049	break;
5050
5051	IEM_NOT_REACHED_DEFAULT_CASE_RET();
5052	}
5053	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
5054	}
5055	return rcStrict;
5056	}
5057
5058
5059
5060	/**
5061	* Stores a data byte.
5062	*
5063	* @returns Strict VBox status code.
5064	* @param pIemCpu The IEM per CPU data.
5065	* @param iSegReg The index of the segment register to use for
5066	* this access. The base and limits are checked.
5067	* @param GCPtrMem The address of the guest memory.
5068	* @param u8Value The value to store.
5069	*/
5070	static VBOXSTRICTRC iemMemStoreDataU8(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
5071	{
5072	/* The lazy approach for now... */
5073	uint8_t *pu8Dst;
5074	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5075	if (rc == VINF_SUCCESS)
5076	{
5077	*pu8Dst = u8Value;
5078	rc = iemMemCommitAndUnmap(pIemCpu, pu8Dst, IEM_ACCESS_DATA_W);
5079	}
5080	return rc;
5081	}
5082
5083
5084	/**
5085	* Stores a data word.
5086	*
5087	* @returns Strict VBox status code.
5088	* @param pIemCpu The IEM per CPU data.
5089	* @param iSegReg The index of the segment register to use for
5090	* this access. The base and limits are checked.
5091	* @param GCPtrMem The address of the guest memory.
5092	* @param u16Value The value to store.
5093	*/
5094	static VBOXSTRICTRC iemMemStoreDataU16(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
5095	{
5096	/* The lazy approach for now... */
5097	uint16_t *pu16Dst;
5098	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5099	if (rc == VINF_SUCCESS)
5100	{
5101	*pu16Dst = u16Value;
5102	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_DATA_W);
5103	}
5104	return rc;
5105	}
5106
5107
5108	/**
5109	* Stores a data dword.
5110	*
5111	* @returns Strict VBox status code.
5112	* @param pIemCpu The IEM per CPU data.
5113	* @param iSegReg The index of the segment register to use for
5114	* this access. The base and limits are checked.
5115	* @param GCPtrMem The address of the guest memory.
5116	* @param u32Value The value to store.
5117	*/
5118	static VBOXSTRICTRC iemMemStoreDataU32(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
5119	{
5120	/* The lazy approach for now... */
5121	uint32_t *pu32Dst;
5122	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5123	if (rc == VINF_SUCCESS)
5124	{
5125	*pu32Dst = u32Value;
5126	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_DATA_W);
5127	}
5128	return rc;
5129	}
5130
5131
5132	/**
5133	* Stores a data qword.
5134	*
5135	* @returns Strict VBox status code.
5136	* @param pIemCpu The IEM per CPU data.
5137	* @param iSegReg The index of the segment register to use for
5138	* this access. The base and limits are checked.
5139	* @param GCPtrMem The address of the guest memory.
5140	* @param u64Value The value to store.
5141	*/
5142	static VBOXSTRICTRC iemMemStoreDataU64(PIEMCPU pIemCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
5143	{
5144	/* The lazy approach for now... */
5145	uint64_t *pu64Dst;
5146	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
5147	if (rc == VINF_SUCCESS)
5148	{
5149	*pu64Dst = u64Value;
5150	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_DATA_W);
5151	}
5152	return rc;
5153	}
5154
5155
5156	/**
5157	* Pushes a word onto the stack.
5158	*
5159	* @returns Strict VBox status code.
5160	* @param pIemCpu The IEM per CPU data.
5161	* @param u16Value The value to push.
5162	*/
5163	static VBOXSTRICTRC iemMemStackPushU16(PIEMCPU pIemCpu, uint16_t u16Value)
5164	{
5165	/* Increment the stack pointer. */
5166	uint64_t uNewRsp;
5167	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5168	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 2, &uNewRsp);
5169
5170	/* Write the word the lazy way. */
5171	uint16_t *pu16Dst;
5172	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5173	if (rc == VINF_SUCCESS)
5174	{
5175	*pu16Dst = u16Value;
5176	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
5177	}
5178
5179	/* Commit the new RSP value unless we an access handler made trouble. */
5180	if (rc == VINF_SUCCESS)
5181	pCtx->rsp = uNewRsp;
5182
5183	return rc;
5184	}
5185
5186
5187	/**
5188	* Pushes a dword onto the stack.
5189	*
5190	* @returns Strict VBox status code.
5191	* @param pIemCpu The IEM per CPU data.
5192	* @param u32Value The value to push.
5193	*/
5194	static VBOXSTRICTRC iemMemStackPushU32(PIEMCPU pIemCpu, uint32_t u32Value)
5195	{
5196	/* Increment the stack pointer. */
5197	uint64_t uNewRsp;
5198	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5199	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 4, &uNewRsp);
5200
5201	/* Write the word the lazy way. */
5202	uint32_t *pu32Dst;
5203	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5204	if (rc == VINF_SUCCESS)
5205	{
5206	*pu32Dst = u32Value;
5207	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
5208	}
5209
5210	/* Commit the new RSP value unless we an access handler made trouble. */
5211	if (rc == VINF_SUCCESS)
5212	pCtx->rsp = uNewRsp;
5213
5214	return rc;
5215	}
5216
5217
5218	/**
5219	* Pushes a qword onto the stack.
5220	*
5221	* @returns Strict VBox status code.
5222	* @param pIemCpu The IEM per CPU data.
5223	* @param u64Value The value to push.
5224	*/
5225	static VBOXSTRICTRC iemMemStackPushU64(PIEMCPU pIemCpu, uint64_t u64Value)
5226	{
5227	/* Increment the stack pointer. */
5228	uint64_t uNewRsp;
5229	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5230	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, 8, &uNewRsp);
5231
5232	/* Write the word the lazy way. */
5233	uint64_t *pu64Dst;
5234	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5235	if (rc == VINF_SUCCESS)
5236	{
5237	*pu64Dst = u64Value;
5238	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
5239	}
5240
5241	/* Commit the new RSP value unless we an access handler made trouble. */
5242	if (rc == VINF_SUCCESS)
5243	pCtx->rsp = uNewRsp;
5244
5245	return rc;
5246	}
5247
5248
5249	/**
5250	* Pops a word from the stack.
5251	*
5252	* @returns Strict VBox status code.
5253	* @param pIemCpu The IEM per CPU data.
5254	* @param pu16Value Where to store the popped value.
5255	*/
5256	static VBOXSTRICTRC iemMemStackPopU16(PIEMCPU pIemCpu, uint16_t *pu16Value)
5257	{
5258	/* Increment the stack pointer. */
5259	uint64_t uNewRsp;
5260	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5261	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 2, &uNewRsp);
5262
5263	/* Write the word the lazy way. */
5264	uint16_t const *pu16Src;
5265	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5266	if (rc == VINF_SUCCESS)
5267	{
5268	pu16Value = pu16Src;
5269	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
5270
5271	/* Commit the new RSP value. */
5272	if (rc == VINF_SUCCESS)
5273	pCtx->rsp = uNewRsp;
5274	}
5275
5276	return rc;
5277	}
5278
5279
5280	/**
5281	* Pops a dword from the stack.
5282	*
5283	* @returns Strict VBox status code.
5284	* @param pIemCpu The IEM per CPU data.
5285	* @param pu32Value Where to store the popped value.
5286	*/
5287	static VBOXSTRICTRC iemMemStackPopU32(PIEMCPU pIemCpu, uint32_t *pu32Value)
5288	{
5289	/* Increment the stack pointer. */
5290	uint64_t uNewRsp;
5291	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5292	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 4, &uNewRsp);
5293
5294	/* Write the word the lazy way. */
5295	uint32_t const *pu32Src;
5296	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5297	if (rc == VINF_SUCCESS)
5298	{
5299	pu32Value = pu32Src;
5300	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
5301
5302	/* Commit the new RSP value. */
5303	if (rc == VINF_SUCCESS)
5304	pCtx->rsp = uNewRsp;
5305	}
5306
5307	return rc;
5308	}
5309
5310
5311	/**
5312	* Pops a qword from the stack.
5313	*
5314	* @returns Strict VBox status code.
5315	* @param pIemCpu The IEM per CPU data.
5316	* @param pu64Value Where to store the popped value.
5317	*/
5318	static VBOXSTRICTRC iemMemStackPopU64(PIEMCPU pIemCpu, uint64_t *pu64Value)
5319	{
5320	/* Increment the stack pointer. */
5321	uint64_t uNewRsp;
5322	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5323	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, 8, &uNewRsp);
5324
5325	/* Write the word the lazy way. */
5326	uint64_t const *pu64Src;
5327	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5328	if (rc == VINF_SUCCESS)
5329	{
5330	pu64Value = pu64Src;
5331	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
5332
5333	/* Commit the new RSP value. */
5334	if (rc == VINF_SUCCESS)
5335	pCtx->rsp = uNewRsp;
5336	}
5337
5338	return rc;
5339	}
5340
5341
5342	/**
5343	* Pushes a word onto the stack, using a temporary stack pointer.
5344	*
5345	* @returns Strict VBox status code.
5346	* @param pIemCpu The IEM per CPU data.
5347	* @param u16Value The value to push.
5348	* @param pTmpRsp Pointer to the temporary stack pointer.
5349	*/
5350	static VBOXSTRICTRC iemMemStackPushU16Ex(PIEMCPU pIemCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
5351	{
5352	/* Increment the stack pointer. */
5353	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5354	RTUINT64U NewRsp = *pTmpRsp;
5355	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 2, pCtx);
5356
5357	/* Write the word the lazy way. */
5358	uint16_t *pu16Dst;
5359	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5360	if (rc == VINF_SUCCESS)
5361	{
5362	*pu16Dst = u16Value;
5363	rc = iemMemCommitAndUnmap(pIemCpu, pu16Dst, IEM_ACCESS_STACK_W);
5364	}
5365
5366	/* Commit the new RSP value unless we an access handler made trouble. */
5367	if (rc == VINF_SUCCESS)
5368	*pTmpRsp = NewRsp;
5369
5370	return rc;
5371	}
5372
5373
5374	/**
5375	* Pushes a dword onto the stack, using a temporary stack pointer.
5376	*
5377	* @returns Strict VBox status code.
5378	* @param pIemCpu The IEM per CPU data.
5379	* @param u32Value The value to push.
5380	* @param pTmpRsp Pointer to the temporary stack pointer.
5381	*/
5382	static VBOXSTRICTRC iemMemStackPushU32Ex(PIEMCPU pIemCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
5383	{
5384	/* Increment the stack pointer. */
5385	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5386	RTUINT64U NewRsp = *pTmpRsp;
5387	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 4, pCtx);
5388
5389	/* Write the word the lazy way. */
5390	uint32_t *pu32Dst;
5391	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5392	if (rc == VINF_SUCCESS)
5393	{
5394	*pu32Dst = u32Value;
5395	rc = iemMemCommitAndUnmap(pIemCpu, pu32Dst, IEM_ACCESS_STACK_W);
5396	}
5397
5398	/* Commit the new RSP value unless we an access handler made trouble. */
5399	if (rc == VINF_SUCCESS)
5400	*pTmpRsp = NewRsp;
5401
5402	return rc;
5403	}
5404
5405
5406	#ifdef SOME_UNUSED_FUNCTION
5407	/**
5408	* Pushes a dword onto the stack, using a temporary stack pointer.
5409	*
5410	* @returns Strict VBox status code.
5411	* @param pIemCpu The IEM per CPU data.
5412	* @param u64Value The value to push.
5413	* @param pTmpRsp Pointer to the temporary stack pointer.
5414	*/
5415	static VBOXSTRICTRC iemMemStackPushU64Ex(PIEMCPU pIemCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
5416	{
5417	/* Increment the stack pointer. */
5418	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5419	RTUINT64U NewRsp = *pTmpRsp;
5420	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(&NewRsp, 8, pCtx);
5421
5422	/* Write the word the lazy way. */
5423	uint64_t *pu64Dst;
5424	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5425	if (rc == VINF_SUCCESS)
5426	{
5427	*pu64Dst = u64Value;
5428	rc = iemMemCommitAndUnmap(pIemCpu, pu64Dst, IEM_ACCESS_STACK_W);
5429	}
5430
5431	/* Commit the new RSP value unless we an access handler made trouble. */
5432	if (rc == VINF_SUCCESS)
5433	*pTmpRsp = NewRsp;
5434
5435	return rc;
5436	}
5437	#endif
5438
5439
5440	/**
5441	* Pops a word from the stack, using a temporary stack pointer.
5442	*
5443	* @returns Strict VBox status code.
5444	* @param pIemCpu The IEM per CPU data.
5445	* @param pu16Value Where to store the popped value.
5446	* @param pTmpRsp Pointer to the temporary stack pointer.
5447	*/
5448	static VBOXSTRICTRC iemMemStackPopU16Ex(PIEMCPU pIemCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
5449	{
5450	/* Increment the stack pointer. */
5451	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5452	RTUINT64U NewRsp = *pTmpRsp;
5453	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 2, pCtx);
5454
5455	/* Write the word the lazy way. */
5456	uint16_t const *pu16Src;
5457	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5458	if (rc == VINF_SUCCESS)
5459	{
5460	pu16Value = pu16Src;
5461	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
5462
5463	/* Commit the new RSP value. */
5464	if (rc == VINF_SUCCESS)
5465	*pTmpRsp = NewRsp;
5466	}
5467
5468	return rc;
5469	}
5470
5471
5472	/**
5473	* Pops a dword from the stack, using a temporary stack pointer.
5474	*
5475	* @returns Strict VBox status code.
5476	* @param pIemCpu The IEM per CPU data.
5477	* @param pu32Value Where to store the popped value.
5478	* @param pTmpRsp Pointer to the temporary stack pointer.
5479	*/
5480	static VBOXSTRICTRC iemMemStackPopU32Ex(PIEMCPU pIemCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
5481	{
5482	/* Increment the stack pointer. */
5483	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5484	RTUINT64U NewRsp = *pTmpRsp;
5485	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 4, pCtx);
5486
5487	/* Write the word the lazy way. */
5488	uint32_t const *pu32Src;
5489	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5490	if (rc == VINF_SUCCESS)
5491	{
5492	pu32Value = pu32Src;
5493	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
5494
5495	/* Commit the new RSP value. */
5496	if (rc == VINF_SUCCESS)
5497	*pTmpRsp = NewRsp;
5498	}
5499
5500	return rc;
5501	}
5502
5503
5504	/**
5505	* Pops a qword from the stack, using a temporary stack pointer.
5506	*
5507	* @returns Strict VBox status code.
5508	* @param pIemCpu The IEM per CPU data.
5509	* @param pu64Value Where to store the popped value.
5510	* @param pTmpRsp Pointer to the temporary stack pointer.
5511	*/
5512	static VBOXSTRICTRC iemMemStackPopU64Ex(PIEMCPU pIemCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
5513	{
5514	/* Increment the stack pointer. */
5515	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5516	RTUINT64U NewRsp = *pTmpRsp;
5517	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 8, pCtx);
5518
5519	/* Write the word the lazy way. */
5520	uint64_t const *pu64Src;
5521	VBOXSTRICTRC rcStrict = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5522	if (rcStrict == VINF_SUCCESS)
5523	{
5524	pu64Value = pu64Src;
5525	rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
5526
5527	/* Commit the new RSP value. */
5528	if (rcStrict == VINF_SUCCESS)
5529	*pTmpRsp = NewRsp;
5530	}
5531
5532	return rcStrict;
5533	}
5534
5535
5536	/**
5537	* Begin a special stack push (used by interrupt, exceptions and such).
5538	*
5539	* This will raise #SS or #PF if appropriate.
5540	*
5541	* @returns Strict VBox status code.
5542	* @param pIemCpu The IEM per CPU data.
5543	* @param cbMem The number of bytes to push onto the stack.
5544	* @param ppvMem Where to return the pointer to the stack memory.
5545	* As with the other memory functions this could be
5546	* direct access or bounce buffered access, so
5547	* don't commit register until the commit call
5548	* succeeds.
5549	* @param puNewRsp Where to return the new RSP value. This must be
5550	* passed unchanged to
5551	* iemMemStackPushCommitSpecial().
5552	*/
5553	static VBOXSTRICTRC iemMemStackPushBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void *ppvMem, uint64_t puNewRsp)
5554	{
5555	Assert(cbMem < UINT8_MAX);
5556	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5557	RTGCPTR GCPtrTop = iemRegGetRspForPush(pCtx, (uint8_t)cbMem, puNewRsp);
5558	return iemMemMap(pIemCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
5559	}
5560
5561
5562	/**
5563	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
5564	*
5565	* This will update the rSP.
5566	*
5567	* @returns Strict VBox status code.
5568	* @param pIemCpu The IEM per CPU data.
5569	* @param pvMem The pointer returned by
5570	* iemMemStackPushBeginSpecial().
5571	* @param uNewRsp The new RSP value returned by
5572	* iemMemStackPushBeginSpecial().
5573	*/
5574	static VBOXSTRICTRC iemMemStackPushCommitSpecial(PIEMCPU pIemCpu, void *pvMem, uint64_t uNewRsp)
5575	{
5576	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, pvMem, IEM_ACCESS_STACK_W);
5577	if (rcStrict == VINF_SUCCESS)
5578	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
5579	return rcStrict;
5580	}
5581
5582
5583	/**
5584	* Begin a special stack pop (used by iret, retf and such).
5585	*
5586	* This will raise \#SS or \#PF if appropriate.
5587	*
5588	* @returns Strict VBox status code.
5589	* @param pIemCpu The IEM per CPU data.
5590	* @param cbMem The number of bytes to push onto the stack.
5591	* @param ppvMem Where to return the pointer to the stack memory.
5592	* @param puNewRsp Where to return the new RSP value. This must be
5593	* passed unchanged to
5594	* iemMemStackPopCommitSpecial() or applied
5595	* manually if iemMemStackPopDoneSpecial() is used.
5596	*/
5597	static VBOXSTRICTRC iemMemStackPopBeginSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
5598	{
5599	Assert(cbMem < UINT8_MAX);
5600	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5601	RTGCPTR GCPtrTop = iemRegGetRspForPop(pCtx, (uint8_t)cbMem, puNewRsp);
5602	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5603	}
5604
5605
5606	/**
5607	* Continue a special stack pop (used by iret and retf).
5608	*
5609	* This will raise \#SS or \#PF if appropriate.
5610	*
5611	* @returns Strict VBox status code.
5612	* @param pIemCpu The IEM per CPU data.
5613	* @param cbMem The number of bytes to push onto the stack.
5614	* @param ppvMem Where to return the pointer to the stack memory.
5615	* @param puNewRsp Where to return the new RSP value. This must be
5616	* passed unchanged to
5617	* iemMemStackPopCommitSpecial() or applied
5618	* manually if iemMemStackPopDoneSpecial() is used.
5619	*/
5620	static VBOXSTRICTRC iemMemStackPopContinueSpecial(PIEMCPU pIemCpu, size_t cbMem, void const *ppvMem, uint64_t puNewRsp)
5621	{
5622	Assert(cbMem < UINT8_MAX);
5623	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5624	RTUINT64U NewRsp;
5625	NewRsp.u = *puNewRsp;
5626	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(&NewRsp, 8, pCtx);
5627	*puNewRsp = NewRsp.u;
5628	return iemMemMap(pIemCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
5629	}
5630
5631
5632	/**
5633	* Commits a special stack pop (started by iemMemStackPopBeginSpecial).
5634	*
5635	* This will update the rSP.
5636	*
5637	* @returns Strict VBox status code.
5638	* @param pIemCpu The IEM per CPU data.
5639	* @param pvMem The pointer returned by
5640	* iemMemStackPopBeginSpecial().
5641	* @param uNewRsp The new RSP value returned by
5642	* iemMemStackPopBeginSpecial().
5643	*/
5644	static VBOXSTRICTRC iemMemStackPopCommitSpecial(PIEMCPU pIemCpu, void const *pvMem, uint64_t uNewRsp)
5645	{
5646	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
5647	if (rcStrict == VINF_SUCCESS)
5648	pIemCpu->CTX_SUFF(pCtx)->rsp = uNewRsp;
5649	return rcStrict;
5650	}
5651
5652
5653	/**
5654	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
5655	* iemMemStackPopContinueSpecial).
5656	*
5657	* The caller will manually commit the rSP.
5658	*
5659	* @returns Strict VBox status code.
5660	* @param pIemCpu The IEM per CPU data.
5661	* @param pvMem The pointer returned by
5662	* iemMemStackPopBeginSpecial() or
5663	* iemMemStackPopContinueSpecial().
5664	*/
5665	static VBOXSTRICTRC iemMemStackPopDoneSpecial(PIEMCPU pIemCpu, void const *pvMem)
5666	{
5667	return iemMemCommitAndUnmap(pIemCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
5668	}
5669
5670
5671	/**
5672	* Fetches a system table dword.
5673	*
5674	* @returns Strict VBox status code.
5675	* @param pIemCpu The IEM per CPU data.
5676	* @param pu32Dst Where to return the dword.
5677	* @param iSegReg The index of the segment register to use for
5678	* this access. The base and limits are checked.
5679	* @param GCPtrMem The address of the guest memory.
5680	*/
5681	static VBOXSTRICTRC iemMemFetchSysU32(PIEMCPU pIemCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
5682	{
5683	/* The lazy approach for now... */
5684	uint32_t const *pu32Src;
5685	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
5686	if (rc == VINF_SUCCESS)
5687	{
5688	pu32Dst = pu32Src;
5689	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
5690	}
5691	return rc;
5692	}
5693
5694
5695	/**
5696	* Fetches a system table qword.
5697	*
5698	* @returns Strict VBox status code.
5699	* @param pIemCpu The IEM per CPU data.
5700	* @param pu64Dst Where to return the qword.
5701	* @param iSegReg The index of the segment register to use for
5702	* this access. The base and limits are checked.
5703	* @param GCPtrMem The address of the guest memory.
5704	*/
5705	static VBOXSTRICTRC iemMemFetchSysU64(PIEMCPU pIemCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
5706	{
5707	/* The lazy approach for now... */
5708	uint64_t const *pu64Src;
5709	VBOXSTRICTRC rc = iemMemMap(pIemCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
5710	if (rc == VINF_SUCCESS)
5711	{
5712	pu64Dst = pu64Src;
5713	rc = iemMemCommitAndUnmap(pIemCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
5714	}
5715	return rc;
5716	}
5717
5718
5719	/**
5720	* Fetches a descriptor table entry.
5721	*
5722	* @returns Strict VBox status code.
5723	* @param pIemCpu The IEM per CPU.
5724	* @param pDesc Where to return the descriptor table entry.
5725	* @param uSel The selector which table entry to fetch.
5726	*/
5727	static VBOXSTRICTRC iemMemFetchSelDesc(PIEMCPU pIemCpu, PIEMSELDESC pDesc, uint16_t uSel)
5728	{
5729	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5730
5731	/** @todo did the 286 require all 8 bytes to be accessible? */
5732	/*
5733	* Get the selector table base and check bounds.
5734	*/
5735	RTGCPTR GCPtrBase;
5736	if (uSel & X86_SEL_LDT)
5737	{
5738	if ( !pCtx->ldtrHid.Attr.n.u1Present
5739	\|\| (uSel \| 0x7U) > pCtx->ldtrHid.u32Limit )
5740	{
5741	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
5742	uSel, pCtx->ldtrHid.u32Limit, pCtx->ldtr));
5743	/** @todo is this the right exception? */
5744	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5745	}
5746
5747	Assert(pCtx->ldtrHid.Attr.n.u1Present);
5748	GCPtrBase = pCtx->ldtrHid.u64Base;
5749	}
5750	else
5751	{
5752	if ((uSel \| 0x7U) > pCtx->gdtr.cbGdt)
5753	{
5754	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pCtx->gdtr.cbGdt));
5755	/** @todo is this the right exception? */
5756	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5757	}
5758	GCPtrBase = pCtx->gdtr.pGdt;
5759	}
5760
5761	/*
5762	* Read the legacy descriptor and maybe the long mode extensions if
5763	* required.
5764	*/
5765	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
5766	if (rcStrict == VINF_SUCCESS)
5767	{
5768	if ( !IEM_IS_LONG_MODE(pIemCpu)
5769	\|\| pDesc->Legacy.Gen.u1DescType)
5770	pDesc->Long.au64[1] = 0;
5771	else if ((uint32_t)(uSel & X86_SEL_MASK) + 15 < (uSel & X86_SEL_LDT ? pCtx->ldtrHid.u32Limit : pCtx->gdtr.cbGdt))
5772	rcStrict = iemMemFetchSysU64(pIemCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
5773	else
5774	{
5775	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
5776	/** @todo is this the right exception? */
5777	return iemRaiseGeneralProtectionFaultBySelector(pIemCpu, uSel);
5778	}
5779	}
5780	return rcStrict;
5781	}
5782
5783
5784	/**
5785	* Fakes a long mode stack selector for SS = 0.
5786	*
5787	* @param pDescSs Where to return the fake stack descriptor.
5788	* @param uDpl The DPL we want.
5789	*/
5790	static void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
5791	{
5792	pDescSs->Long.au64[0] = 0;
5793	pDescSs->Long.au64[1] = 0;
5794	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
5795	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
5796	pDescSs->Long.Gen.u2Dpl = uDpl;
5797	pDescSs->Long.Gen.u1Present = 1;
5798	pDescSs->Long.Gen.u1Long = 1;
5799	}
5800
5801
5802	/**
5803	* Marks the selector descriptor as accessed (only non-system descriptors).
5804	*
5805	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
5806	* will therefore skip the limit checks.
5807	*
5808	* @returns Strict VBox status code.
5809	* @param pIemCpu The IEM per CPU.
5810	* @param uSel The selector.
5811	*/
5812	static VBOXSTRICTRC iemMemMarkSelDescAccessed(PIEMCPU pIemCpu, uint16_t uSel)
5813	{
5814	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
5815
5816	/*
5817	* Get the selector table base and calculate the entry address.
5818	*/
5819	RTGCPTR GCPtr = uSel & X86_SEL_LDT
5820	? pCtx->ldtrHid.u64Base
5821	: pCtx->gdtr.pGdt;
5822	GCPtr += uSel & X86_SEL_MASK;
5823
5824	/*
5825	* ASMAtomicBitSet will assert if the address is misaligned, so do some
5826	* ugly stuff to avoid this. This will make sure it's an atomic access
5827	* as well more or less remove any question about 8-bit or 32-bit accesss.
5828	*/
5829	VBOXSTRICTRC rcStrict;
5830	uint32_t volatile *pu32;
5831	if ((GCPtr & 3) == 0)
5832	{
5833	/* The normal case, map the 32-bit bits around the accessed bit (40). */
5834	GCPtr += 2 + 2;
5835	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
5836	if (rcStrict != VINF_SUCCESS)
5837	return rcStrict;
5838	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
5839	}
5840	else
5841	{
5842	/* The misaligned GDT/LDT case, map the whole thing. */
5843	rcStrict = iemMemMap(pIemCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
5844	if (rcStrict != VINF_SUCCESS)
5845	return rcStrict;
5846	switch ((uintptr_t)pu32 & 3)
5847	{
5848	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
5849	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
5850	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
5851	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
5852	}
5853	}
5854
5855	return iemMemCommitAndUnmap(pIemCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
5856	}
5857
5858	/** @} */
5859
5860
5861	/*
5862	* Include the C/C++ implementation of instruction.
5863	*/
5864	#include "IEMAllCImpl.cpp.h"
5865
5866
5867
5868	/** @name "Microcode" macros.
5869	*
5870	* The idea is that we should be able to use the same code to interpret
5871	* instructions as well as recompiler instructions. Thus this obfuscation.
5872	*
5873	* @{
5874	*/
5875	#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
5876	#define IEM_MC_END() }
5877	#define IEM_MC_PAUSE() do {} while (0)
5878	#define IEM_MC_CONTINUE() do {} while (0)
5879
5880	/** Internal macro. */
5881	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
5882	do \
5883	{ \
5884	VBOXSTRICTRC rcStrict2 = a_Expr; \
5885	if (rcStrict2 != VINF_SUCCESS) \
5886	return rcStrict2; \
5887	} while (0)
5888
5889	#define IEM_MC_ADVANCE_RIP() iemRegUpdateRip(pIemCpu)
5890	#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pIemCpu, a_i8))
5891	#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pIemCpu, a_i16))
5892	#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pIemCpu, a_i32))
5893	#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u16NewIP)))
5894	#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u32NewIP)))
5895	#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pIemCpu), (a_u64NewIP)))
5896
5897	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pIemCpu)
5898	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
5899	do { \
5900	if ((pIemCpu)->CTX_SUFF(pCtx)->cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
5901	return iemRaiseDeviceNotAvailable(pIemCpu); \
5902	} while (0)
5903	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
5904	do { \
5905	if ((pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW & X86_FSW_ES) \
5906	return iemRaiseMathFault(pIemCpu); \
5907	} while (0)
5908	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
5909	do { \
5910	if (pIemCpu->uCpl != 0) \
5911	return iemRaiseGeneralProtectionFault0(pIemCpu); \
5912	} while (0)
5913
5914
5915	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
5916	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
5917	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
5918	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
5919	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
5920	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
5921	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
5922	uint32_t a_Name; \
5923	uint32_t *a_pName = &a_Name
5924	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
5925	do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u = (a_EFlags); Assert((pIemCpu)->CTX_SUFF(pCtx)->eflags.u & X86_EFL_1); } while (0)
5926
5927	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
5928	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
5929
5930	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5931	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5932	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5933	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pIemCpu, (a_iGReg))
5934	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5935	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5936	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pIemCpu, (a_iGReg))
5937	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5938	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5939	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pIemCpu, (a_iGReg))
5940	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
5941	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pIemCpu, (a_iGReg))
5942	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
5943	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pIemCpu, (a_iGReg))
5944	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pIemCpu, (a_iGReg))
5945	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pIemCpu, (a_iGReg))
5946	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
5947	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) (a_u16Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5948	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) (a_u32Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5949	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) (a_u64Dst) = iemSRegFetchU16(pIemCpu, (a_iSReg))
5950	#define IEM_MC_FETCH_CR0_U16(a_u16Dst) (a_u16Dst) = (uint16_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
5951	#define IEM_MC_FETCH_CR0_U32(a_u32Dst) (a_u32Dst) = (uint32_t)(pIemCpu)->CTX_SUFF(pCtx)->cr0
5952	#define IEM_MC_FETCH_CR0_U64(a_u64Dst) (a_u64Dst) = (pIemCpu)->CTX_SUFF(pCtx)->cr0
5953	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = (pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5954	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5955	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pIemCpu->CTX_SUFF(pCtx)->fpu.FSW
5956	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pIemCpu->CTX_SUFF(pCtx)->fpu.FCW
5957
5958	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8(pIemCpu, (a_iGReg)) = (a_u8Value)
5959	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u16Value)
5960	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
5961	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) = (a_u64Value)
5962	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
5963	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
5964	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
5965	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
5966	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= UINT32_MAX
5967	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
5968	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
5969	do { pIemCpu->CTX_SUFF(pCtx)->fpu.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
5970
5971	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8(pIemCpu, (a_iGReg))
5972	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t *)iemGRegRef(pIemCpu, (a_iGReg))
5973	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
5974	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
5975	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t *)iemGRegRef(pIemCpu, (a_iGReg))
5976	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t *)iemGRegRef(pIemCpu, (a_iGReg))
5977	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &(pIemCpu)->CTX_SUFF(pCtx)->eflags.u
5978
5979	#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u8Value)
5980	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u16Value)
5981	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
5982	do { \
5983	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5984	*pu32Reg += (a_u32Value); \
5985	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5986	} while (0)
5987	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) += (a_u64Value)
5988
5989	#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u8Value)
5990	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u16Value)
5991	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
5992	do { \
5993	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
5994	*pu32Reg -= (a_u32Value); \
5995	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
5996	} while (0)
5997	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) -= (a_u64Value)
5998
5999	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pIemCpu, (a_iGReg)); } while (0)
6000	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pIemCpu, (a_iGReg)); } while (0)
6001	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pIemCpu, (a_iGReg)); } while (0)
6002	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pIemCpu, (a_iGReg)); } while (0)
6003	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
6004	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
6005	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
6006
6007	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
6008	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
6009	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
6010	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
6011
6012	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
6013	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
6014	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
6015
6016	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
6017	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
6018
6019	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
6020	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
6021	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
6022
6023	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
6024	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
6025	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
6026
6027	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
6028
6029	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
6030
6031	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u8Value)
6032	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u16Value)
6033	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
6034	do { \
6035	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
6036	*pu32Reg &= (a_u32Value); \
6037	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
6038	} while (0)
6039	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) &= (a_u64Value)
6040
6041	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) (uint8_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u8Value)
6042	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) (uint16_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u16Value)
6043	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
6044	do { \
6045	uint32_t pu32Reg = (uint32_t )iemGRegRef(pIemCpu, (a_iGReg)); \
6046	*pu32Reg \|= (a_u32Value); \
6047	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
6048	} while (0)
6049	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) (uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) \|= (a_u64Value)
6050
6051
6052	#define IEM_MC_SET_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u \|= (a_fBit); } while (0)
6053	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u &= ~(a_fBit); } while (0)
6054	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { (pIemCpu)->CTX_SUFF(pCtx)->eflags.u ^= (a_fBit); } while (0)
6055
6056	#define IEM_MC_CLEAR_FSW_EX() do { (pIemCpu)->CTX_SUFF(pCtx)->fpu.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
6057
6058
6059	#define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
6060	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
6061	#define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
6062	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
6063	#define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
6064	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
6065
6066	#define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
6067	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
6068	#define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
6069	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
6070	#define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
6071	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
6072
6073	#define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6074	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
6075	#define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
6076	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
6077	#define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
6078	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
6079
6080	#define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6081	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
6082
6083	#define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6084	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
6085	#define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
6086	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
6087
6088	#define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
6089	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
6090	#define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
6091	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pIemCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
6092	#define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
6093	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pIemCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
6094
6095
6096	#define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
6097	do { \
6098	uint8_t u8Tmp; \
6099	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6100	(a_u16Dst) = u8Tmp; \
6101	} while (0)
6102	#define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6103	do { \
6104	uint8_t u8Tmp; \
6105	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6106	(a_u32Dst) = u8Tmp; \
6107	} while (0)
6108	#define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6109	do { \
6110	uint8_t u8Tmp; \
6111	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6112	(a_u64Dst) = u8Tmp; \
6113	} while (0)
6114	#define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6115	do { \
6116	uint16_t u16Tmp; \
6117	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6118	(a_u32Dst) = u16Tmp; \
6119	} while (0)
6120	#define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6121	do { \
6122	uint16_t u16Tmp; \
6123	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6124	(a_u64Dst) = u16Tmp; \
6125	} while (0)
6126	#define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6127	do { \
6128	uint32_t u32Tmp; \
6129	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
6130	(a_u64Dst) = u32Tmp; \
6131	} while (0)
6132
6133	#define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
6134	do { \
6135	uint8_t u8Tmp; \
6136	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6137	(a_u16Dst) = (int8_t)u8Tmp; \
6138	} while (0)
6139	#define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6140	do { \
6141	uint8_t u8Tmp; \
6142	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6143	(a_u32Dst) = (int8_t)u8Tmp; \
6144	} while (0)
6145	#define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6146	do { \
6147	uint8_t u8Tmp; \
6148	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pIemCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
6149	(a_u64Dst) = (int8_t)u8Tmp; \
6150	} while (0)
6151	#define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
6152	do { \
6153	uint16_t u16Tmp; \
6154	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6155	(a_u32Dst) = (int16_t)u16Tmp; \
6156	} while (0)
6157	#define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6158	do { \
6159	uint16_t u16Tmp; \
6160	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pIemCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
6161	(a_u64Dst) = (int16_t)u16Tmp; \
6162	} while (0)
6163	#define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
6164	do { \
6165	uint32_t u32Tmp; \
6166	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pIemCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
6167	(a_u64Dst) = (int32_t)u32Tmp; \
6168	} while (0)
6169
6170	#define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
6171	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
6172	#define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
6173	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
6174	#define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
6175	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
6176	#define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
6177	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
6178
6179	#define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
6180	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
6181	#define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
6182	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
6183	#define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
6184	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
6185	#define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
6186	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pIemCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
6187
6188	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
6189	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
6190	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
6191	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
6192	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u32 = UINT32_C(0xffc00000)
6193	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
6194	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
6195	do { \
6196	(a_pr80Dst)->au64[1] = UINT64_C(0xc000000000000000); \
6197	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
6198	} while (0)
6199
6200
6201	#define IEM_MC_PUSH_U16(a_u16Value) \
6202	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pIemCpu, (a_u16Value)))
6203	#define IEM_MC_PUSH_U32(a_u32Value) \
6204	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pIemCpu, (a_u32Value)))
6205	#define IEM_MC_PUSH_U64(a_u64Value) \
6206	IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pIemCpu, (a_u64Value)))
6207
6208	#define IEM_MC_POP_U16(a_pu16Value) \
6209	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pIemCpu, (a_pu16Value)))
6210	#define IEM_MC_POP_U32(a_pu32Value) \
6211	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pIemCpu, (a_pu32Value)))
6212	#define IEM_MC_POP_U64(a_pu64Value) \
6213	IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pIemCpu, (a_pu64Value)))
6214
6215	/** Maps guest memory for direct or bounce buffered access.
6216	* The purpose is to pass it to an operand implementation, thus the a_iArg.
6217	* @remarks May return.
6218	*/
6219	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
6220	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
6221
6222	/** Maps guest memory for direct or bounce buffered access.
6223	* The purpose is to pass it to an operand implementation, thus the a_iArg.
6224	* @remarks May return.
6225	*/
6226	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
6227	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pIemCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
6228
6229	/** Commits the memory and unmaps the guest memory.
6230	* @remarks May return.
6231	*/
6232	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
6233	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess)))
6234
6235	/** Commits the memory and unmaps the guest memory unless the FPU status word
6236	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
6237	* that would cause FLD not to store.
6238	*
6239	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
6240	* store, while \#P will not.
6241	*
6242	* @remarks May in theory return - for now.
6243	*/
6244	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
6245	do { \
6246	if ( !(a_u16FSW & X86_FSW_ES) \
6247	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
6248	& ~(pIemCpu->CTX_SUFF(pCtx)->fpu.FCW & X86_FCW_MASK_ALL) ) ) \
6249	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pIemCpu, (a_pvMem), (a_fAccess))); \
6250	} while (0)
6251
6252	/** Calculate efficient address from R/M. */
6253	#define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm) \
6254	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pIemCpu, (bRm), &(a_GCPtrEff)))
6255
6256	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
6257	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
6258	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
6259	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
6260	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
6261
6262	/**
6263	* Defers the rest of the instruction emulation to a C implementation routine
6264	* and returns, only taking the standard parameters.
6265	*
6266	* @param a_pfnCImpl The pointer to the C routine.
6267	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
6268	*/
6269	#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
6270
6271	/**
6272	* Defers the rest of instruction emulation to a C implementation routine and
6273	* returns, taking one argument in addition to the standard ones.
6274	*
6275	* @param a_pfnCImpl The pointer to the C routine.
6276	* @param a0 The argument.
6277	*/
6278	#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
6279
6280	/**
6281	* Defers the rest of the instruction emulation to a C implementation routine
6282	* and returns, taking two arguments in addition to the standard ones.
6283	*
6284	* @param a_pfnCImpl The pointer to the C routine.
6285	* @param a0 The first extra argument.
6286	* @param a1 The second extra argument.
6287	*/
6288	#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
6289
6290	/**
6291	* Defers the rest of the instruction emulation to a C implementation routine
6292	* and returns, taking two arguments in addition to the standard ones.
6293	*
6294	* @param a_pfnCImpl The pointer to the C routine.
6295	* @param a0 The first extra argument.
6296	* @param a1 The second extra argument.
6297	* @param a2 The third extra argument.
6298	*/
6299	#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
6300
6301	/**
6302	* Defers the rest of the instruction emulation to a C implementation routine
6303	* and returns, taking two arguments in addition to the standard ones.
6304	*
6305	* @param a_pfnCImpl The pointer to the C routine.
6306	* @param a0 The first extra argument.
6307	* @param a1 The second extra argument.
6308	* @param a2 The third extra argument.
6309	* @param a3 The fourth extra argument.
6310	* @param a4 The fifth extra argument.
6311	*/
6312	#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2, a3, a4)
6313
6314	/**
6315	* Defers the entire instruction emulation to a C implementation routine and
6316	* returns, only taking the standard parameters.
6317	*
6318	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6319	*
6320	* @param a_pfnCImpl The pointer to the C routine.
6321	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
6322	*/
6323	#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode)
6324
6325	/**
6326	* Defers the entire instruction emulation to a C implementation routine and
6327	* returns, taking one argument in addition to the standard ones.
6328	*
6329	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6330	*
6331	* @param a_pfnCImpl The pointer to the C routine.
6332	* @param a0 The argument.
6333	*/
6334	#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0)
6335
6336	/**
6337	* Defers the entire instruction emulation to a C implementation routine and
6338	* returns, taking two arguments in addition to the standard ones.
6339	*
6340	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6341	*
6342	* @param a_pfnCImpl The pointer to the C routine.
6343	* @param a0 The first extra argument.
6344	* @param a1 The second extra argument.
6345	*/
6346	#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1)
6347
6348	/**
6349	* Defers the entire instruction emulation to a C implementation routine and
6350	* returns, taking three arguments in addition to the standard ones.
6351	*
6352	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
6353	*
6354	* @param a_pfnCImpl The pointer to the C routine.
6355	* @param a0 The first extra argument.
6356	* @param a1 The second extra argument.
6357	* @param a2 The third extra argument.
6358	*/
6359	#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pIemCpu, pIemCpu->offOpcode, a0, a1, a2)
6360
6361	/**
6362	* Calls a FPU assembly implementation taking one visible argument.
6363	*
6364	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6365	* @param a0 The first extra argument.
6366	*/
6367	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
6368	do { \
6369	iemFpuPrepareUsage(pIemCpu); \
6370	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0)); \
6371	} while (0)
6372
6373	/**
6374	* Calls a FPU assembly implementation taking two visible arguments.
6375	*
6376	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6377	* @param a0 The first extra argument.
6378	* @param a1 The second extra argument.
6379	*/
6380	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
6381	do { \
6382	iemFpuPrepareUsage(pIemCpu); \
6383	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1)); \
6384	} while (0)
6385
6386	/**
6387	* Calls a FPU assembly implementation taking three visible arguments.
6388	*
6389	* @param a_pfnAImpl Pointer to the assembly FPU routine.
6390	* @param a0 The first extra argument.
6391	* @param a1 The second extra argument.
6392	* @param a2 The third extra argument.
6393	*/
6394	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
6395	do { \
6396	iemFpuPrepareUsage(pIemCpu); \
6397	a_pfnAImpl(&pIemCpu->CTX_SUFF(pCtx)->fpu, (a0), (a1), (a2)); \
6398	} while (0)
6399
6400	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
6401	do { \
6402	(a_FpuData).FSW = (a_FSW); \
6403	(a_FpuData).r80Result = *(a_pr80Value); \
6404	} while (0)
6405
6406	/** Pushes FPU result onto the stack. */
6407	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
6408	iemFpuPushResult(pIemCpu, &a_FpuData)
6409	/** Pushes FPU result onto the stack and sets the FPUDP. */
6410	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
6411	iemFpuPushResultWithMemOp(pIemCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
6412
6413	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
6414	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
6415	iemFpuPushResultTwo(pIemCpu, &a_FpuDataTwo)
6416
6417	/** Stores FPU result in a stack register. */
6418	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
6419	iemFpuStoreResult(pIemCpu, &a_FpuData, a_iStReg)
6420	/** Stores FPU result in a stack register and pops the stack. */
6421	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
6422	iemFpuStoreResultThenPop(pIemCpu, &a_FpuData, a_iStReg)
6423	/** Stores FPU result in a stack register and sets the FPUDP. */
6424	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
6425	iemFpuStoreResultWithMemOp(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
6426	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
6427	* stack. */
6428	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
6429	iemFpuStoreResultWithMemOpThenPop(pIemCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
6430
6431	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
6432	#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
6433	iemFpuUpdateOpcodeAndIp(pIemCpu)
6434	/** Free a stack register (for FFREE and FFREEP). */
6435	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
6436	iemFpuStackFree(pIemCpu, a_iStReg)
6437	/** Increment the FPU stack pointer. */
6438	#define IEM_MC_FPU_STACK_INC_TOP() \
6439	iemFpuStackIncTop(pIemCpu)
6440	/** Decrement the FPU stack pointer. */
6441	#define IEM_MC_FPU_STACK_DEC_TOP() \
6442	iemFpuStackDecTop(pIemCpu)
6443
6444	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
6445	#define IEM_MC_UPDATE_FSW(a_u16FSW) \
6446	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
6447	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
6448	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
6449	iemFpuUpdateFSW(pIemCpu, a_u16FSW)
6450	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
6451	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
6452	iemFpuUpdateFSWWithMemOp(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
6453	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
6454	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
6455	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
6456	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
6457	* stack. */
6458	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
6459	iemFpuUpdateFSWWithMemOpThenPop(pIemCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
6460	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
6461	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
6462	iemFpuUpdateFSWThenPop(pIemCpu, a_u16FSW)
6463
6464	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
6465	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
6466	iemFpuStackUnderflow(pIemCpu, a_iStDst)
6467	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
6468	* stack. */
6469	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
6470	iemFpuStackUnderflowThenPop(pIemCpu, a_iStDst)
6471	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
6472	* FPUDS. */
6473	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
6474	iemFpuStackUnderflowWithMemOp(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
6475	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
6476	* FPUDS. Pops stack. */
6477	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
6478	iemFpuStackUnderflowWithMemOpThenPop(pIemCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
6479	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
6480	* stack twice. */
6481	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
6482	iemFpuStackUnderflowThenPopPop(pIemCpu)
6483	/** Raises a FPU stack underflow exception for an instruction pushing a result
6484	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
6485	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
6486	iemFpuStackPushUnderflow(pIemCpu)
6487	/** Raises a FPU stack underflow exception for an instruction pushing a result
6488	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
6489	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
6490	iemFpuStackPushUnderflowTwo(pIemCpu)
6491
6492	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
6493	* FPUIP, FPUCS and FOP. */
6494	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
6495	iemFpuStackPushOverflow(pIemCpu)
6496	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
6497	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
6498	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
6499	iemFpuStackPushOverflowWithMemOp(pIemCpu, a_iEffSeg, a_GCPtrEff)
6500
6501	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) {
6502	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit))) {
6503	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits)) {
6504	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBits))) {
6505	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
6506	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6507	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6508	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
6509	if ( !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6510	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6511	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
6512	if ( (pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
6513	\|\| !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6514	!= !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6515	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
6516	if ( !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit)) \
6517	&& !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit1)) \
6518	== !!(pIemCpu->CTX_SUFF(pCtx)->eflags.u & (a_fBit2)) ) {
6519	#define IEM_MC_IF_CX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->cx != 0) {
6520	#define IEM_MC_IF_ECX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->ecx != 0) {
6521	#define IEM_MC_IF_RCX_IS_NZ() if (pIemCpu->CTX_SUFF(pCtx)->rcx != 0) {
6522	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6523	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
6524	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6525	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6526	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
6527	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6528	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
6529	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
6530	&& (pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6531	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6532	if ( pIemCpu->CTX_SUFF(pCtx)->cx != 0 \
6533	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6534	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6535	if ( pIemCpu->CTX_SUFF(pCtx)->ecx != 0 \
6536	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6537	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
6538	if ( pIemCpu->CTX_SUFF(pCtx)->rcx != 0 \
6539	&& !(pIemCpu->CTX_SUFF(pCtx)->eflags.u & a_fBit)) {
6540	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
6541	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if ((uint64_t )iemGRegRef(pIemCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
6542	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
6543	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) == VINF_SUCCESS) {
6544	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
6545	if (iemFpuStRegNotEmpty(pIemCpu, (a_iSt)) != VINF_SUCCESS) {
6546	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
6547	if (iemFpuStRegNotEmptyRef(pIemCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
6548	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
6549	if (iemFpu2StRegsNotEmptyRef(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
6550	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
6551	if (iemFpu2StRegsNotEmptyRefFirst(pIemCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
6552	#define IEM_MC_IF_FCW_IM() \
6553	if (pIemCpu->CTX_SUFF(pCtx)->fpu.FCW & X86_FCW_IM) {
6554
6555	#define IEM_MC_ELSE() } else {
6556	#define IEM_MC_ENDIF() } do {} while (0)
6557
6558	/** @} */
6559
6560
6561	/** @name Opcode Debug Helpers.
6562	* @{
6563	*/
6564	#ifdef DEBUG
6565	# define IEMOP_MNEMONIC(a_szMnemonic) \
6566	Log4(("decode - %04x:%RGv %s%s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip, \
6567	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pIemCpu->cInstructions))
6568	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) \
6569	Log4(("decode - %04x:%RGv %s%s %s [#%u]\n", pIemCpu->CTX_SUFF(pCtx)->cs, pIemCpu->CTX_SUFF(pCtx)->rip, \
6570	pIemCpu->fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, a_szOps, pIemCpu->cInstructions))
6571	#else
6572	# define IEMOP_MNEMONIC(a_szMnemonic) do { } while (0)
6573	# define IEMOP_MNEMONIC2(a_szMnemonic, a_szOps) do { } while (0)
6574	#endif
6575
6576	/** @} */
6577
6578
6579	/** @name Opcode Helpers.
6580	* @{
6581	*/
6582
6583	/** The instruction allows no lock prefixing (in this encoding), throw #UD if
6584	* lock prefixed.
6585	* @deprecated IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX */
6586	#define IEMOP_HLP_NO_LOCK_PREFIX() \
6587	do \
6588	{ \
6589	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
6590	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
6591	} while (0)
6592
6593	/** The instruction is not available in 64-bit mode, throw #UD if we're in
6594	* 64-bit mode. */
6595	#define IEMOP_HLP_NO_64BIT() \
6596	do \
6597	{ \
6598	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
6599	return IEMOP_RAISE_INVALID_OPCODE(); \
6600	} while (0)
6601
6602	/** The instruction defaults to 64-bit operand size if 64-bit mode. */
6603	#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
6604	do \
6605	{ \
6606	if (pIemCpu->enmCpuMode == IEMMODE_64BIT) \
6607	iemRecalEffOpSize64Default(pIemCpu); \
6608	} while (0)
6609
6610	/**
6611	* Done decoding.
6612	*/
6613	#define IEMOP_HLP_DONE_DECODING() \
6614	do \
6615	{ \
6616	/nothing for now, maybe later... / \
6617	} while (0)
6618
6619	/**
6620	* Done decoding, raise \#UD exception if lock prefix present.
6621	*/
6622	#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
6623	do \
6624	{ \
6625	if (pIemCpu->fPrefixes & IEM_OP_PRF_LOCK) \
6626	return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
6627	} while (0)
6628
6629
6630	/**
6631	* Calculates the effective address of a ModR/M memory operand.
6632	*
6633	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
6634	*
6635	* @return Strict VBox status code.
6636	* @param pIemCpu The IEM per CPU data.
6637	* @param bRm The ModRM byte.
6638	* @param pGCPtrEff Where to return the effective address.
6639	*/
6640	static VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PIEMCPU pIemCpu, uint8_t bRm, PRTGCPTR pGCPtrEff)
6641	{
6642	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
6643	PCCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
6644	#define SET_SS_DEF() \
6645	do \
6646	{ \
6647	if (!(pIemCpu->fPrefixes & IEM_OP_PRF_SEG_MASK)) \
6648	pIemCpu->iEffSeg = X86_SREG_SS; \
6649	} while (0)
6650
6651	/** @todo Check the effective address size crap! */
6652	switch (pIemCpu->enmEffAddrMode)
6653	{
6654	case IEMMODE_16BIT:
6655	{
6656	uint16_t u16EffAddr;
6657
6658	/* Handle the disp16 form with no registers first. */
6659	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
6660	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
6661	else
6662	{
6663	/* Get the displacment. */
6664	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6665	{
6666	case 0: u16EffAddr = 0; break;
6667	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
6668	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
6669	default: AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
6670	}
6671
6672	/* Add the base and index registers to the disp. */
6673	switch (bRm & X86_MODRM_RM_MASK)
6674	{
6675	case 0: u16EffAddr += pCtx->bx + pCtx->si; break;
6676	case 1: u16EffAddr += pCtx->bx + pCtx->di; break;
6677	case 2: u16EffAddr += pCtx->bp + pCtx->si; SET_SS_DEF(); break;
6678	case 3: u16EffAddr += pCtx->bp + pCtx->di; SET_SS_DEF(); break;
6679	case 4: u16EffAddr += pCtx->si; break;
6680	case 5: u16EffAddr += pCtx->di; break;
6681	case 6: u16EffAddr += pCtx->bp; SET_SS_DEF(); break;
6682	case 7: u16EffAddr += pCtx->bx; break;
6683	}
6684	}
6685
6686	*pGCPtrEff = u16EffAddr;
6687	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#06RGv\n", *pGCPtrEff));
6688	return VINF_SUCCESS;
6689	}
6690
6691	case IEMMODE_32BIT:
6692	{
6693	uint32_t u32EffAddr;
6694
6695	/* Handle the disp32 form with no registers first. */
6696	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
6697	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
6698	else
6699	{
6700	/* Get the register (or SIB) value. */
6701	switch ((bRm & X86_MODRM_RM_MASK))
6702	{
6703	case 0: u32EffAddr = pCtx->eax; break;
6704	case 1: u32EffAddr = pCtx->ecx; break;
6705	case 2: u32EffAddr = pCtx->edx; break;
6706	case 3: u32EffAddr = pCtx->ebx; break;
6707	case 4: /* SIB */
6708	{
6709	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
6710
6711	/* Get the index and scale it. */
6712	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
6713	{
6714	case 0: u32EffAddr = pCtx->eax; break;
6715	case 1: u32EffAddr = pCtx->ecx; break;
6716	case 2: u32EffAddr = pCtx->edx; break;
6717	case 3: u32EffAddr = pCtx->ebx; break;
6718	case 4: u32EffAddr = 0; /none / break;
6719	case 5: u32EffAddr = pCtx->ebp; break;
6720	case 6: u32EffAddr = pCtx->esi; break;
6721	case 7: u32EffAddr = pCtx->edi; break;
6722	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6723	}
6724	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
6725
6726	/* add base */
6727	switch (bSib & X86_SIB_BASE_MASK)
6728	{
6729	case 0: u32EffAddr += pCtx->eax; break;
6730	case 1: u32EffAddr += pCtx->ecx; break;
6731	case 2: u32EffAddr += pCtx->edx; break;
6732	case 3: u32EffAddr += pCtx->ebx; break;
6733	case 4: u32EffAddr += pCtx->esp; SET_SS_DEF(); break;
6734	case 5:
6735	if ((bRm & X86_MODRM_MOD_MASK) != 0)
6736	{
6737	u32EffAddr += pCtx->ebp;
6738	SET_SS_DEF();
6739	}
6740	else
6741	{
6742	uint32_t u32Disp;
6743	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6744	u32EffAddr += u32Disp;
6745	}
6746	break;
6747	case 6: u32EffAddr += pCtx->esi; break;
6748	case 7: u32EffAddr += pCtx->edi; break;
6749	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6750	}
6751	break;
6752	}
6753	case 5: u32EffAddr = pCtx->ebp; SET_SS_DEF(); break;
6754	case 6: u32EffAddr = pCtx->esi; break;
6755	case 7: u32EffAddr = pCtx->edi; break;
6756	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6757	}
6758
6759	/* Get and add the displacement. */
6760	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6761	{
6762	case 0:
6763	break;
6764	case 1:
6765	{
6766	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
6767	u32EffAddr += i8Disp;
6768	break;
6769	}
6770	case 2:
6771	{
6772	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6773	u32EffAddr += u32Disp;
6774	break;
6775	}
6776	default:
6777	AssertFailedReturn(VERR_INTERNAL_ERROR_2); /* (caller checked for these) */
6778	}
6779
6780	}
6781	if (pIemCpu->enmEffAddrMode == IEMMODE_32BIT)
6782	*pGCPtrEff = u32EffAddr;
6783	else
6784	{
6785	Assert(pIemCpu->enmEffAddrMode == IEMMODE_16BIT);
6786	*pGCPtrEff = u32EffAddr & UINT16_MAX;
6787	}
6788	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
6789	return VINF_SUCCESS;
6790	}
6791
6792	case IEMMODE_64BIT:
6793	{
6794	uint64_t u64EffAddr;
6795
6796	/* Handle the rip+disp32 form with no registers first. */
6797	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
6798	{
6799	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
6800	u64EffAddr += pCtx->rip + pIemCpu->offOpcode;
6801	}
6802	else
6803	{
6804	/* Get the register (or SIB) value. */
6805	switch ((bRm & X86_MODRM_RM_MASK) \| pIemCpu->uRexB)
6806	{
6807	case 0: u64EffAddr = pCtx->rax; break;
6808	case 1: u64EffAddr = pCtx->rcx; break;
6809	case 2: u64EffAddr = pCtx->rdx; break;
6810	case 3: u64EffAddr = pCtx->rbx; break;
6811	case 5: u64EffAddr = pCtx->rbp; SET_SS_DEF(); break;
6812	case 6: u64EffAddr = pCtx->rsi; break;
6813	case 7: u64EffAddr = pCtx->rdi; break;
6814	case 8: u64EffAddr = pCtx->r8; break;
6815	case 9: u64EffAddr = pCtx->r9; break;
6816	case 10: u64EffAddr = pCtx->r10; break;
6817	case 11: u64EffAddr = pCtx->r11; break;
6818	case 13: u64EffAddr = pCtx->r13; break;
6819	case 14: u64EffAddr = pCtx->r14; break;
6820	case 15: u64EffAddr = pCtx->r15; break;
6821	/* SIB */
6822	case 4:
6823	case 12:
6824	{
6825	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
6826
6827	/* Get the index and scale it. */
6828	switch (((bSib & X86_SIB_INDEX_SHIFT) >> X86_SIB_INDEX_SMASK) \| pIemCpu->uRexIndex)
6829	{
6830	case 0: u64EffAddr = pCtx->rax; break;
6831	case 1: u64EffAddr = pCtx->rcx; break;
6832	case 2: u64EffAddr = pCtx->rdx; break;
6833	case 3: u64EffAddr = pCtx->rbx; break;
6834	case 4: u64EffAddr = 0; /none / break;
6835	case 5: u64EffAddr = pCtx->rbp; break;
6836	case 6: u64EffAddr = pCtx->rsi; break;
6837	case 7: u64EffAddr = pCtx->rdi; break;
6838	case 8: u64EffAddr = pCtx->r8; break;
6839	case 9: u64EffAddr = pCtx->r9; break;
6840	case 10: u64EffAddr = pCtx->r10; break;
6841	case 11: u64EffAddr = pCtx->r11; break;
6842	case 12: u64EffAddr = pCtx->r12; break;
6843	case 13: u64EffAddr = pCtx->r13; break;
6844	case 14: u64EffAddr = pCtx->r14; break;
6845	case 15: u64EffAddr = pCtx->r15; break;
6846	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6847	}
6848	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
6849
6850	/* add base */
6851	switch ((bSib & X86_SIB_BASE_MASK) \| pIemCpu->uRexB)
6852	{
6853	case 0: u64EffAddr += pCtx->rax; break;
6854	case 1: u64EffAddr += pCtx->rcx; break;
6855	case 2: u64EffAddr += pCtx->rdx; break;
6856	case 3: u64EffAddr += pCtx->rbx; break;
6857	case 4: u64EffAddr += pCtx->rsp; SET_SS_DEF(); break;
6858	case 6: u64EffAddr += pCtx->rsi; break;
6859	case 7: u64EffAddr += pCtx->rdi; break;
6860	case 8: u64EffAddr += pCtx->r8; break;
6861	case 9: u64EffAddr += pCtx->r9; break;
6862	case 10: u64EffAddr += pCtx->r10; break;
6863	case 11: u64EffAddr += pCtx->r11; break;
6864	case 14: u64EffAddr += pCtx->r14; break;
6865	case 15: u64EffAddr += pCtx->r15; break;
6866	/* complicated encodings */
6867	case 5:
6868	case 13:
6869	if ((bRm & X86_MODRM_MOD_MASK) != 0)
6870	{
6871	if (!pIemCpu->uRexB)
6872	{
6873	u64EffAddr += pCtx->rbp;
6874	SET_SS_DEF();
6875	}
6876	else
6877	u64EffAddr += pCtx->r13;
6878	}
6879	else
6880	{
6881	uint32_t u32Disp;
6882	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6883	u64EffAddr += (int32_t)u32Disp;
6884	}
6885	break;
6886	}
6887	break;
6888	}
6889	IEM_NOT_REACHED_DEFAULT_CASE_RET();
6890	}
6891
6892	/* Get and add the displacement. */
6893	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
6894	{
6895	case 0:
6896	break;
6897	case 1:
6898	{
6899	int8_t i8Disp;
6900	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
6901	u64EffAddr += i8Disp;
6902	break;
6903	}
6904	case 2:
6905	{
6906	uint32_t u32Disp;
6907	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
6908	u64EffAddr += (int32_t)u32Disp;
6909	break;
6910	}
6911	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
6912	}
6913
6914	}
6915	if (pIemCpu->enmEffAddrMode == IEMMODE_64BIT)
6916	*pGCPtrEff = u64EffAddr;
6917	else
6918	*pGCPtrEff = u64EffAddr & UINT16_MAX;
6919	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
6920	return VINF_SUCCESS;
6921	}
6922	}
6923
6924	AssertFailedReturn(VERR_INTERNAL_ERROR_3);
6925	}
6926
6927	/** @} */
6928
6929
6930
6931	/*
6932	* Include the instructions
6933	*/
6934	#include "IEMAllInstructions.cpp.h"
6935
6936
6937
6938
6939	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
6940
6941	/**
6942	* Sets up execution verification mode.
6943	*/
6944	static void iemExecVerificationModeSetup(PIEMCPU pIemCpu)
6945	{
6946	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
6947	PCPUMCTX pOrgCtx = pIemCpu->CTX_SUFF(pCtx);
6948
6949	/*
6950	* Enable verification and/or logging.
6951	*/
6952	pIemCpu->fNoRem = !LogIs6Enabled(); /* logging triggers the no-rem/rem verification stuff */
6953	if ( pIemCpu->fNoRem
6954	#if 0 /* auto enable on first paged protected mode interrupt */
6955	&& pOrgCtx->eflags.Bits.u1IF
6956	&& (pOrgCtx->cr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG)
6957	&& TRPMHasTrap(pVCpu)
6958	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
6959	#endif
6960	#if 0
6961	&& pOrgCtx->cs == 0x10
6962	&& ( pOrgCtx->rip == 0x90119e3e
6963	\|\| pOrgCtx->rip == 0x901d9810
6964	)
6965	#endif
6966	#if 0 /* Auto enable DSL - FPU stuff. */
6967	&& pOrgCtx->cs == 0x10
6968	&& (// pOrgCtx->rip == 0xc02ec07f
6969	//\|\| pOrgCtx->rip == 0xc02ec082
6970	//\|\| pOrgCtx->rip == 0xc02ec0c9
6971	0
6972	\|\| pOrgCtx->rip == 0x0c010e7c4 /* fxsave */
6973	)
6974	#endif
6975	#if 1 /* Auto enable DSL - fstp st0 stuff. */
6976	&& pOrgCtx->cs == 0x23
6977	&& pOrgCtx->rip == 0x804aff7
6978	#endif
6979	#if 0
6980	&& pOrgCtx->rip == 0x9022bb3a
6981	#endif
6982	#if 0
6983	&& 0
6984	#endif
6985	)
6986	{
6987	RTLogGroupSettings(NULL, "iem.eo.l6.l2");
6988	RTLogFlags(NULL, "enabled");
6989	pIemCpu->fNoRem = false;
6990	}
6991
6992	/*
6993	* Switch state.
6994	*/
6995	if (IEM_VERIFICATION_ENABLED(pIemCpu))
6996	{
6997	static CPUMCTX s_DebugCtx; /* Ugly! */
6998
6999	s_DebugCtx = *pOrgCtx;
7000	pIemCpu->CTX_SUFF(pCtx) = &s_DebugCtx;
7001	}
7002
7003	/*
7004	* See if there is an interrupt pending in TRPM and inject it if we can.
7005	*/
7006	if ( pOrgCtx->eflags.Bits.u1IF
7007	&& TRPMHasTrap(pVCpu)
7008	&& EMGetInhibitInterruptsPC(pVCpu) != pOrgCtx->rip)
7009	{
7010	uint8_t u8TrapNo;
7011	TRPMEVENT enmType;
7012	RTGCUINT uErrCode;
7013	RTGCPTR uCr2;
7014	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2); AssertRC(rc2);
7015	IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2);
7016	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
7017	TRPMResetTrap(pVCpu);
7018	}
7019
7020	/*
7021	* Reset the counters.
7022	*/
7023	pIemCpu->cIOReads = 0;
7024	pIemCpu->cIOWrites = 0;
7025	pIemCpu->fUndefinedEFlags = 0;
7026
7027	if (IEM_VERIFICATION_ENABLED(pIemCpu))
7028	{
7029	/*
7030	* Free all verification records.
7031	*/
7032	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pIemEvtRecHead;
7033	pIemCpu->pIemEvtRecHead = NULL;
7034	pIemCpu->ppIemEvtRecNext = &pIemCpu->pIemEvtRecHead;
7035	do
7036	{
7037	while (pEvtRec)
7038	{
7039	PIEMVERIFYEVTREC pNext = pEvtRec->pNext;
7040	pEvtRec->pNext = pIemCpu->pFreeEvtRec;
7041	pIemCpu->pFreeEvtRec = pEvtRec;
7042	pEvtRec = pNext;
7043	}
7044	pEvtRec = pIemCpu->pOtherEvtRecHead;
7045	pIemCpu->pOtherEvtRecHead = NULL;
7046	pIemCpu->ppOtherEvtRecNext = &pIemCpu->pOtherEvtRecHead;
7047	} while (pEvtRec);
7048	}
7049	}
7050
7051
7052	/**
7053	* Allocate an event record.
7054	* @returns Poitner to a record.
7055	*/
7056	static PIEMVERIFYEVTREC iemVerifyAllocRecord(PIEMCPU pIemCpu)
7057	{
7058	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
7059	return NULL;
7060
7061	PIEMVERIFYEVTREC pEvtRec = pIemCpu->pFreeEvtRec;
7062	if (pEvtRec)
7063	pIemCpu->pFreeEvtRec = pEvtRec->pNext;
7064	else
7065	{
7066	if (!pIemCpu->ppIemEvtRecNext)
7067	return NULL; /* Too early (fake PCIBIOS), ignore notification. */
7068
7069	pEvtRec = (PIEMVERIFYEVTREC)MMR3HeapAlloc(IEMCPU_TO_VM(pIemCpu), MM_TAG_EM /* lazy bird/, sizeof(pEvtRec));
7070	if (!pEvtRec)
7071	return NULL;
7072	}
7073	pEvtRec->enmEvent = IEMVERIFYEVENT_INVALID;
7074	pEvtRec->pNext = NULL;
7075	return pEvtRec;
7076	}
7077
7078
7079	/**
7080	* IOMMMIORead notification.
7081	*/
7082	VMM_INT_DECL(void) IEMNotifyMMIORead(PVM pVM, RTGCPHYS GCPhys, size_t cbValue)
7083	{
7084	PVMCPU pVCpu = VMMGetCpu(pVM);
7085	if (!pVCpu)
7086	return;
7087	PIEMCPU pIemCpu = &pVCpu->iem.s;
7088	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7089	if (!pEvtRec)
7090	return;
7091	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_READ;
7092	pEvtRec->u.RamRead.GCPhys = GCPhys;
7093	pEvtRec->u.RamRead.cb = (uint32_t)cbValue;
7094	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7095	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7096	}
7097
7098
7099	/**
7100	* IOMMMIOWrite notification.
7101	*/
7102	VMM_INT_DECL(void) IEMNotifyMMIOWrite(PVM pVM, RTGCPHYS GCPhys, uint32_t u32Value, size_t cbValue)
7103	{
7104	PVMCPU pVCpu = VMMGetCpu(pVM);
7105	if (!pVCpu)
7106	return;
7107	PIEMCPU pIemCpu = &pVCpu->iem.s;
7108	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7109	if (!pEvtRec)
7110	return;
7111	pEvtRec->enmEvent = IEMVERIFYEVENT_RAM_WRITE;
7112	pEvtRec->u.RamWrite.GCPhys = GCPhys;
7113	pEvtRec->u.RamWrite.cb = (uint32_t)cbValue;
7114	pEvtRec->u.RamWrite.ab[0] = RT_BYTE1(u32Value);
7115	pEvtRec->u.RamWrite.ab[1] = RT_BYTE2(u32Value);
7116	pEvtRec->u.RamWrite.ab[2] = RT_BYTE3(u32Value);
7117	pEvtRec->u.RamWrite.ab[3] = RT_BYTE4(u32Value);
7118	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7119	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7120	}
7121
7122
7123	/**
7124	* IOMIOPortRead notification.
7125	*/
7126	VMM_INT_DECL(void) IEMNotifyIOPortRead(PVM pVM, RTIOPORT Port, size_t cbValue)
7127	{
7128	PVMCPU pVCpu = VMMGetCpu(pVM);
7129	if (!pVCpu)
7130	return;
7131	PIEMCPU pIemCpu = &pVCpu->iem.s;
7132	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7133	if (!pEvtRec)
7134	return;
7135	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
7136	pEvtRec->u.IOPortRead.Port = Port;
7137	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
7138	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7139	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7140	}
7141
7142	/**
7143	* IOMIOPortWrite notification.
7144	*/
7145	VMM_INT_DECL(void) IEMNotifyIOPortWrite(PVM pVM, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7146	{
7147	PVMCPU pVCpu = VMMGetCpu(pVM);
7148	if (!pVCpu)
7149	return;
7150	PIEMCPU pIemCpu = &pVCpu->iem.s;
7151	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7152	if (!pEvtRec)
7153	return;
7154	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
7155	pEvtRec->u.IOPortWrite.Port = Port;
7156	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
7157	pEvtRec->u.IOPortWrite.u32Value = u32Value;
7158	pEvtRec->pNext = *pIemCpu->ppOtherEvtRecNext;
7159	*pIemCpu->ppOtherEvtRecNext = pEvtRec;
7160	}
7161
7162
7163	VMM_INT_DECL(void) IEMNotifyIOPortReadString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrDst, RTGCUINTREG cTransfers, size_t cbValue)
7164	{
7165	AssertFailed();
7166	}
7167
7168
7169	VMM_INT_DECL(void) IEMNotifyIOPortWriteString(PVM pVM, RTIOPORT Port, RTGCPTR GCPtrSrc, RTGCUINTREG cTransfers, size_t cbValue)
7170	{
7171	AssertFailed();
7172	}
7173
7174
7175	/**
7176	* Fakes and records an I/O port read.
7177	*
7178	* @returns VINF_SUCCESS.
7179	* @param pIemCpu The IEM per CPU data.
7180	* @param Port The I/O port.
7181	* @param pu32Value Where to store the fake value.
7182	* @param cbValue The size of the access.
7183	*/
7184	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
7185	{
7186	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7187	if (pEvtRec)
7188	{
7189	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_READ;
7190	pEvtRec->u.IOPortRead.Port = Port;
7191	pEvtRec->u.IOPortRead.cbValue = (uint32_t)cbValue;
7192	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
7193	*pIemCpu->ppIemEvtRecNext = pEvtRec;
7194	}
7195	pIemCpu->cIOReads++;
7196	*pu32Value = 0xcccccccc;
7197	return VINF_SUCCESS;
7198	}
7199
7200
7201	/**
7202	* Fakes and records an I/O port write.
7203	*
7204	* @returns VINF_SUCCESS.
7205	* @param pIemCpu The IEM per CPU data.
7206	* @param Port The I/O port.
7207	* @param u32Value The value being written.
7208	* @param cbValue The size of the access.
7209	*/
7210	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7211	{
7212	PIEMVERIFYEVTREC pEvtRec = iemVerifyAllocRecord(pIemCpu);
7213	if (pEvtRec)
7214	{
7215	pEvtRec->enmEvent = IEMVERIFYEVENT_IOPORT_WRITE;
7216	pEvtRec->u.IOPortWrite.Port = Port;
7217	pEvtRec->u.IOPortWrite.cbValue = (uint32_t)cbValue;
7218	pEvtRec->u.IOPortWrite.u32Value = u32Value;
7219	pEvtRec->pNext = *pIemCpu->ppIemEvtRecNext;
7220	*pIemCpu->ppIemEvtRecNext = pEvtRec;
7221	}
7222	pIemCpu->cIOWrites++;
7223	return VINF_SUCCESS;
7224	}
7225
7226
7227	/**
7228	* Used to add extra details about a stub case.
7229	* @param pIemCpu The IEM per CPU state.
7230	*/
7231	static void iemVerifyAssertMsg2(PIEMCPU pIemCpu)
7232	{
7233	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7234	PVM pVM = IEMCPU_TO_VM(pIemCpu);
7235	PVMCPU pVCpu = IEMCPU_TO_VMCPU(pIemCpu);
7236	char szRegs[4096];
7237	DBGFR3RegPrintf(pVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
7238	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
7239	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
7240	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
7241	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
7242	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
7243	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
7244	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
7245	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
7246	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
7247	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
7248	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
7249	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
7250	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
7251	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
7252	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
7253	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
7254	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
7255	" efer=%016VR{efer}\n"
7256	" pat=%016VR{pat}\n"
7257	" sf_mask=%016VR{sf_mask}\n"
7258	"krnl_gs_base=%016VR{krnl_gs_base}\n"
7259	" lstar=%016VR{lstar}\n"
7260	" star=%016VR{star} cstar=%016VR{cstar}\n"
7261	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
7262	);
7263
7264	char szInstr1[256];
7265	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, pCtx->cs, pCtx->rip - pIemCpu->offOpcode,
7266	DBGF_DISAS_FLAGS_DEFAULT_MODE,
7267	szInstr1, sizeof(szInstr1), NULL);
7268	char szInstr2[256];
7269	DBGFR3DisasInstrEx(pVM, pVCpu->idCpu, 0, 0,
7270	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
7271	szInstr2, sizeof(szInstr2), NULL);
7272
7273	RTAssertMsg2Weak("%s%s\n%s\n", szRegs, szInstr1, szInstr2);
7274	}
7275
7276
7277	/**
7278	* Used by iemVerifyAssertRecord and iemVerifyAssertRecords to add a record
7279	* dump to the assertion info.
7280	*
7281	* @param pEvtRec The record to dump.
7282	*/
7283	static void iemVerifyAssertAddRecordDump(PIEMVERIFYEVTREC pEvtRec)
7284	{
7285	switch (pEvtRec->enmEvent)
7286	{
7287	case IEMVERIFYEVENT_IOPORT_READ:
7288	RTAssertMsg2Add("I/O PORT READ from %#6x, %d bytes\n",
7289	pEvtRec->u.IOPortWrite.Port,
7290	pEvtRec->u.IOPortWrite.cbValue);
7291	break;
7292	case IEMVERIFYEVENT_IOPORT_WRITE:
7293	RTAssertMsg2Add("I/O PORT WRITE to %#6x, %d bytes, value %#x\n",
7294	pEvtRec->u.IOPortWrite.Port,
7295	pEvtRec->u.IOPortWrite.cbValue,
7296	pEvtRec->u.IOPortWrite.u32Value);
7297	break;
7298	case IEMVERIFYEVENT_RAM_READ:
7299	RTAssertMsg2Add("RAM READ at %RGp, %#4zx bytes\n",
7300	pEvtRec->u.RamRead.GCPhys,
7301	pEvtRec->u.RamRead.cb);
7302	break;
7303	case IEMVERIFYEVENT_RAM_WRITE:
7304	RTAssertMsg2Add("RAM WRITE at %RGp, %#4zx bytes: %.*Rhxs\n",
7305	pEvtRec->u.RamWrite.GCPhys,
7306	pEvtRec->u.RamWrite.cb,
7307	(int)pEvtRec->u.RamWrite.cb,
7308	pEvtRec->u.RamWrite.ab);
7309	break;
7310	default:
7311	AssertMsgFailed(("Invalid event type %d\n", pEvtRec->enmEvent));
7312	break;
7313	}
7314	}
7315
7316
7317	/**
7318	* Raises an assertion on the specified record, showing the given message with
7319	* a record dump attached.
7320	*
7321	* @param pIemCpu The IEM per CPU data.
7322	* @param pEvtRec1 The first record.
7323	* @param pEvtRec2 The second record.
7324	* @param pszMsg The message explaining why we're asserting.
7325	*/
7326	static void iemVerifyAssertRecords(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec1, PIEMVERIFYEVTREC pEvtRec2, const char *pszMsg)
7327	{
7328	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
7329	iemVerifyAssertAddRecordDump(pEvtRec1);
7330	iemVerifyAssertAddRecordDump(pEvtRec2);
7331	iemVerifyAssertMsg2(pIemCpu);
7332	RTAssertPanic();
7333	}
7334
7335
7336	/**
7337	* Raises an assertion on the specified record, showing the given message with
7338	* a record dump attached.
7339	*
7340	* @param pIemCpu The IEM per CPU data.
7341	* @param pEvtRec1 The first record.
7342	* @param pszMsg The message explaining why we're asserting.
7343	*/
7344	static void iemVerifyAssertRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec, const char *pszMsg)
7345	{
7346	RTAssertMsg1(pszMsg, __LINE__, __FILE__, __PRETTY_FUNCTION__);
7347	iemVerifyAssertAddRecordDump(pEvtRec);
7348	iemVerifyAssertMsg2(pIemCpu);
7349	RTAssertPanic();
7350	}
7351
7352
7353	/**
7354	* Verifies a write record.
7355	*
7356	* @param pIemCpu The IEM per CPU data.
7357	* @param pEvtRec The write record.
7358	*/
7359	static void iemVerifyWriteRecord(PIEMCPU pIemCpu, PIEMVERIFYEVTREC pEvtRec)
7360	{
7361	uint8_t abBuf[sizeof(pEvtRec->u.RamWrite.ab)]; RT_ZERO(abBuf);
7362	Assert(sizeof(abBuf) >= pEvtRec->u.RamWrite.cb);
7363	int rc = PGMPhysSimpleReadGCPhys(IEMCPU_TO_VM(pIemCpu), abBuf, pEvtRec->u.RamWrite.GCPhys, pEvtRec->u.RamWrite.cb);
7364	if ( RT_FAILURE(rc)
7365	\|\| memcmp(abBuf, pEvtRec->u.RamWrite.ab, pEvtRec->u.RamWrite.cb) )
7366	{
7367	/* fend off ins */
7368	if ( !pIemCpu->cIOReads
7369	\|\| pEvtRec->u.RamWrite.ab[0] != 0xcc
7370	\|\| ( pEvtRec->u.RamWrite.cb != 1
7371	&& pEvtRec->u.RamWrite.cb != 2
7372	&& pEvtRec->u.RamWrite.cb != 4) )
7373	{
7374	/* fend off ROMs */
7375	if ( pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000c0000) > UINT32_C(0x8000)
7376	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0x000e0000) > UINT32_C(0x20000)
7377	&& pEvtRec->u.RamWrite.GCPhys - UINT32_C(0xfffc0000) > UINT32_C(0x40000) )
7378	{
7379	/* fend off fxsave */
7380	if (pEvtRec->u.RamWrite.cb != 512)
7381	{
7382	RTAssertMsg1(NULL, __LINE__, __FILE__, __PRETTY_FUNCTION__);
7383	RTAssertMsg2Weak("Memory at %RGv differs\n", pEvtRec->u.RamWrite.GCPhys);
7384	RTAssertMsg2Add("REM: %.*Rhxs\n"
7385	"IEM: %.*Rhxs\n",
7386	pEvtRec->u.RamWrite.cb, abBuf,
7387	pEvtRec->u.RamWrite.cb, pEvtRec->u.RamWrite.ab);
7388	iemVerifyAssertAddRecordDump(pEvtRec);
7389	iemVerifyAssertMsg2(pIemCpu);
7390	RTAssertPanic();
7391	}
7392	}
7393	}
7394	}
7395
7396	}
7397
7398	/**
7399	* Performs the post-execution verfication checks.
7400	*/
7401	static void iemExecVerificationModeCheck(PIEMCPU pIemCpu)
7402	{
7403	if (!IEM_VERIFICATION_ENABLED(pIemCpu))
7404	return;
7405
7406	/*
7407	* Switch back the state.
7408	*/
7409	PCPUMCTX pOrgCtx = CPUMQueryGuestCtxPtr(IEMCPU_TO_VMCPU(pIemCpu));
7410	PCPUMCTX pDebugCtx = pIemCpu->CTX_SUFF(pCtx);
7411	Assert(pOrgCtx != pDebugCtx);
7412	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
7413
7414	/*
7415	* Execute the instruction in REM.
7416	*/
7417	PVM pVM = IEMCPU_TO_VM(pIemCpu);
7418	EMRemLock(pVM);
7419	int rc = REMR3EmulateInstruction(pVM, IEMCPU_TO_VMCPU(pIemCpu));
7420	AssertRC(rc);
7421	EMRemUnlock(pVM);
7422
7423	/*
7424	* Compare the register states.
7425	*/
7426	unsigned cDiffs = 0;
7427	if (memcmp(pOrgCtx, pDebugCtx, sizeof(*pDebugCtx)))
7428	{
7429	Log(("REM and IEM ends up with different registers!\n"));
7430
7431	# define CHECK_FIELD(a_Field) \
7432	do \
7433	{ \
7434	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
7435	{ \
7436	switch (sizeof(pOrgCtx->a_Field)) \
7437	{ \
7438	case 1: RTAssertMsg2Weak(" %8s differs - iem=%02x - rem=%02x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7439	case 2: RTAssertMsg2Weak(" %8s differs - iem=%04x - rem=%04x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7440	case 4: RTAssertMsg2Weak(" %8s differs - iem=%08x - rem=%08x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7441	case 8: RTAssertMsg2Weak(" %8s differs - iem=%016llx - rem=%016llx\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); break; \
7442	default: RTAssertMsg2Weak(" %8s differs\n", #a_Field); break; \
7443	} \
7444	cDiffs++; \
7445	} \
7446	} while (0)
7447
7448	# define CHECK_BIT_FIELD(a_Field) \
7449	do \
7450	{ \
7451	if (pOrgCtx->a_Field != pDebugCtx->a_Field) \
7452	{ \
7453	RTAssertMsg2Weak(" %8s differs - iem=%02x - rem=%02x\n", #a_Field, pDebugCtx->a_Field, pOrgCtx->a_Field); \
7454	cDiffs++; \
7455	} \
7456	} while (0)
7457
7458	# define CHECK_SEL(a_Sel) \
7459	do \
7460	{ \
7461	CHECK_FIELD(a_Sel); \
7462	if ( pOrgCtx->a_Sel##Hid.Attr.u != pDebugCtx->a_Sel##Hid.Attr.u \
7463	&& (pOrgCtx->a_Sel##Hid.Attr.u \| X86_SEL_TYPE_ACCESSED) != pDebugCtx->a_Sel##Hid.Attr.u) \
7464	{ \
7465	RTAssertMsg2Weak(" %8sHid.Attr differs - iem=%02x - rem=%02x\n", #a_Sel, pDebugCtx->a_Sel##Hid.Attr.u, pOrgCtx->a_Sel##Hid.Attr.u); \
7466	cDiffs++; \
7467	} \
7468	CHECK_FIELD(a_Sel##Hid.u64Base); \
7469	CHECK_FIELD(a_Sel##Hid.u32Limit); \
7470	} while (0)
7471
7472	#if 1 /* The recompiler doesn't update these the intel way. */
7473	pOrgCtx->fpu.FOP = pDebugCtx->fpu.FOP;
7474	pOrgCtx->fpu.FPUIP = pDebugCtx->fpu.FPUIP;
7475	pOrgCtx->fpu.CS = pDebugCtx->fpu.CS;
7476	pOrgCtx->fpu.Rsrvd1 = pDebugCtx->fpu.Rsrvd1;
7477	pOrgCtx->fpu.FPUDP = pDebugCtx->fpu.FPUDP;
7478	pOrgCtx->fpu.DS = pDebugCtx->fpu.DS;
7479	pOrgCtx->fpu.Rsrvd2 = pDebugCtx->fpu.Rsrvd2;
7480	pOrgCtx->fpu.MXCSR_MASK = pDebugCtx->fpu.MXCSR_MASK; /* only for the time being - old snapshots here. */
7481	if ((pOrgCtx->fpu.FSW & X86_FSW_TOP_MASK) == (pDebugCtx->fpu.FSW & X86_FSW_TOP_MASK))
7482	pOrgCtx->fpu.FSW = pDebugCtx->fpu.FSW;
7483	#endif
7484	if (memcmp(&pOrgCtx->fpu, &pDebugCtx->fpu, sizeof(pDebugCtx->fpu)))
7485	{
7486	RTAssertMsg2Weak(" the FPU state differs\n");
7487	cDiffs++;
7488	CHECK_FIELD(fpu.FCW);
7489	CHECK_FIELD(fpu.FSW);
7490	CHECK_FIELD(fpu.FTW);
7491	CHECK_FIELD(fpu.FOP);
7492	CHECK_FIELD(fpu.FPUIP);
7493	CHECK_FIELD(fpu.CS);
7494	CHECK_FIELD(fpu.Rsrvd1);
7495	CHECK_FIELD(fpu.FPUDP);
7496	CHECK_FIELD(fpu.DS);
7497	CHECK_FIELD(fpu.Rsrvd2);
7498	CHECK_FIELD(fpu.MXCSR);
7499	CHECK_FIELD(fpu.MXCSR_MASK);
7500	CHECK_FIELD(fpu.aRegs[0].au64[0]); CHECK_FIELD(fpu.aRegs[0].au64[1]);
7501	CHECK_FIELD(fpu.aRegs[1].au64[0]); CHECK_FIELD(fpu.aRegs[1].au64[1]);
7502	CHECK_FIELD(fpu.aRegs[2].au64[0]); CHECK_FIELD(fpu.aRegs[2].au64[1]);
7503	CHECK_FIELD(fpu.aRegs[3].au64[0]); CHECK_FIELD(fpu.aRegs[3].au64[1]);
7504	CHECK_FIELD(fpu.aRegs[4].au64[0]); CHECK_FIELD(fpu.aRegs[4].au64[1]);
7505	CHECK_FIELD(fpu.aRegs[5].au64[0]); CHECK_FIELD(fpu.aRegs[5].au64[1]);
7506	CHECK_FIELD(fpu.aRegs[6].au64[0]); CHECK_FIELD(fpu.aRegs[6].au64[1]);
7507	CHECK_FIELD(fpu.aRegs[7].au64[0]); CHECK_FIELD(fpu.aRegs[7].au64[1]);
7508	CHECK_FIELD(fpu.aXMM[ 0].au64[0]); CHECK_FIELD(fpu.aXMM[ 0].au64[1]);
7509	CHECK_FIELD(fpu.aXMM[ 1].au64[0]); CHECK_FIELD(fpu.aXMM[ 1].au64[1]);
7510	CHECK_FIELD(fpu.aXMM[ 2].au64[0]); CHECK_FIELD(fpu.aXMM[ 2].au64[1]);
7511	CHECK_FIELD(fpu.aXMM[ 3].au64[0]); CHECK_FIELD(fpu.aXMM[ 3].au64[1]);
7512	CHECK_FIELD(fpu.aXMM[ 4].au64[0]); CHECK_FIELD(fpu.aXMM[ 4].au64[1]);
7513	CHECK_FIELD(fpu.aXMM[ 5].au64[0]); CHECK_FIELD(fpu.aXMM[ 5].au64[1]);
7514	CHECK_FIELD(fpu.aXMM[ 6].au64[0]); CHECK_FIELD(fpu.aXMM[ 6].au64[1]);
7515	CHECK_FIELD(fpu.aXMM[ 7].au64[0]); CHECK_FIELD(fpu.aXMM[ 7].au64[1]);
7516	CHECK_FIELD(fpu.aXMM[ 8].au64[0]); CHECK_FIELD(fpu.aXMM[ 8].au64[1]);
7517	CHECK_FIELD(fpu.aXMM[ 9].au64[0]); CHECK_FIELD(fpu.aXMM[ 9].au64[1]);
7518	CHECK_FIELD(fpu.aXMM[10].au64[0]); CHECK_FIELD(fpu.aXMM[10].au64[1]);
7519	CHECK_FIELD(fpu.aXMM[11].au64[0]); CHECK_FIELD(fpu.aXMM[11].au64[1]);
7520	CHECK_FIELD(fpu.aXMM[12].au64[0]); CHECK_FIELD(fpu.aXMM[12].au64[1]);
7521	CHECK_FIELD(fpu.aXMM[13].au64[0]); CHECK_FIELD(fpu.aXMM[13].au64[1]);
7522	CHECK_FIELD(fpu.aXMM[14].au64[0]); CHECK_FIELD(fpu.aXMM[14].au64[1]);
7523	CHECK_FIELD(fpu.aXMM[15].au64[0]); CHECK_FIELD(fpu.aXMM[15].au64[1]);
7524	for (unsigned i = 0; i < RT_ELEMENTS(pOrgCtx->fpu.au32RsrvdRest); i++)
7525	CHECK_FIELD(fpu.au32RsrvdRest[i]);
7526	}
7527	CHECK_FIELD(rip);
7528	uint32_t fFlagsMask = UINT32_MAX & ~pIemCpu->fUndefinedEFlags;
7529	if ((pOrgCtx->rflags.u & fFlagsMask) != (pDebugCtx->rflags.u & fFlagsMask))
7530	{
7531	RTAssertMsg2Weak(" rflags differs - iem=%08llx rem=%08llx\n", pDebugCtx->rflags.u, pOrgCtx->rflags.u);
7532	CHECK_BIT_FIELD(rflags.Bits.u1CF);
7533	CHECK_BIT_FIELD(rflags.Bits.u1Reserved0);
7534	CHECK_BIT_FIELD(rflags.Bits.u1PF);
7535	CHECK_BIT_FIELD(rflags.Bits.u1Reserved1);
7536	CHECK_BIT_FIELD(rflags.Bits.u1AF);
7537	CHECK_BIT_FIELD(rflags.Bits.u1Reserved2);
7538	CHECK_BIT_FIELD(rflags.Bits.u1ZF);
7539	CHECK_BIT_FIELD(rflags.Bits.u1SF);
7540	CHECK_BIT_FIELD(rflags.Bits.u1TF);
7541	CHECK_BIT_FIELD(rflags.Bits.u1IF);
7542	CHECK_BIT_FIELD(rflags.Bits.u1DF);
7543	CHECK_BIT_FIELD(rflags.Bits.u1OF);
7544	CHECK_BIT_FIELD(rflags.Bits.u2IOPL);
7545	CHECK_BIT_FIELD(rflags.Bits.u1NT);
7546	CHECK_BIT_FIELD(rflags.Bits.u1Reserved3);
7547	CHECK_BIT_FIELD(rflags.Bits.u1RF);
7548	CHECK_BIT_FIELD(rflags.Bits.u1VM);
7549	CHECK_BIT_FIELD(rflags.Bits.u1AC);
7550	CHECK_BIT_FIELD(rflags.Bits.u1VIF);
7551	CHECK_BIT_FIELD(rflags.Bits.u1VIP);
7552	CHECK_BIT_FIELD(rflags.Bits.u1ID);
7553	}
7554
7555	if (pIemCpu->cIOReads != 1 && !pIemCpu->fIgnoreRaxRdx)
7556	CHECK_FIELD(rax);
7557	CHECK_FIELD(rcx);
7558	if (!pIemCpu->fIgnoreRaxRdx)
7559	CHECK_FIELD(rdx);
7560	CHECK_FIELD(rbx);
7561	CHECK_FIELD(rsp);
7562	CHECK_FIELD(rbp);
7563	CHECK_FIELD(rsi);
7564	CHECK_FIELD(rdi);
7565	CHECK_FIELD(r8);
7566	CHECK_FIELD(r9);
7567	CHECK_FIELD(r10);
7568	CHECK_FIELD(r11);
7569	CHECK_FIELD(r12);
7570	CHECK_FIELD(r13);
7571	CHECK_SEL(cs);
7572	CHECK_SEL(ss);
7573	CHECK_SEL(ds);
7574	CHECK_SEL(es);
7575	CHECK_SEL(fs);
7576	CHECK_SEL(gs);
7577	CHECK_FIELD(cr0);
7578	CHECK_FIELD(cr2);
7579	CHECK_FIELD(cr3);
7580	CHECK_FIELD(cr4);
7581	CHECK_FIELD(dr[0]);
7582	CHECK_FIELD(dr[1]);
7583	CHECK_FIELD(dr[2]);
7584	CHECK_FIELD(dr[3]);
7585	CHECK_FIELD(dr[6]);
7586	if ((pOrgCtx->dr[7] & ~X86_DR7_MB1_MASK) != (pDebugCtx->dr[7] & ~X86_DR7_MB1_MASK)) /* REM 'mov drX,greg' bug.*/
7587	CHECK_FIELD(dr[7]);
7588	CHECK_FIELD(gdtr.cbGdt);
7589	CHECK_FIELD(gdtr.pGdt);
7590	CHECK_FIELD(idtr.cbIdt);
7591	CHECK_FIELD(idtr.pIdt);
7592	CHECK_FIELD(ldtr);
7593	CHECK_FIELD(ldtrHid.u64Base);
7594	CHECK_FIELD(ldtrHid.u32Limit);
7595	CHECK_FIELD(ldtrHid.Attr.u);
7596	CHECK_FIELD(tr);
7597	CHECK_FIELD(trHid.u64Base);
7598	CHECK_FIELD(trHid.u32Limit);
7599	CHECK_FIELD(trHid.Attr.u);
7600	CHECK_FIELD(SysEnter.cs);
7601	CHECK_FIELD(SysEnter.eip);
7602	CHECK_FIELD(SysEnter.esp);
7603	CHECK_FIELD(msrEFER);
7604	CHECK_FIELD(msrSTAR);
7605	CHECK_FIELD(msrPAT);
7606	CHECK_FIELD(msrLSTAR);
7607	CHECK_FIELD(msrCSTAR);
7608	CHECK_FIELD(msrSFMASK);
7609	CHECK_FIELD(msrKERNELGSBASE);
7610
7611	if (cDiffs != 0)
7612	{
7613	if (LogIs3Enabled())
7614	DBGFR3Info(pVM, "cpumguest", "verbose", NULL);
7615	RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__);
7616	iemVerifyAssertMsg2(pIemCpu);
7617	RTAssertPanic();
7618	}
7619	# undef CHECK_FIELD
7620	# undef CHECK_BIT_FIELD
7621	}
7622
7623	/*
7624	* If the register state compared fine, check the verification event
7625	* records.
7626	*/
7627	if (cDiffs == 0)
7628	{
7629	/*
7630	* Compare verficiation event records.
7631	* - I/O port accesses should be a 1:1 match.
7632	*/
7633	PIEMVERIFYEVTREC pIemRec = pIemCpu->pIemEvtRecHead;
7634	PIEMVERIFYEVTREC pOtherRec = pIemCpu->pOtherEvtRecHead;
7635	while (pIemRec && pOtherRec)
7636	{
7637	/* Since we might miss RAM writes and reads, ignore reads and check
7638	that any written memory is the same extra ones. */
7639	while ( IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent)
7640	&& !IEMVERIFYEVENT_IS_RAM(pOtherRec->enmEvent)
7641	&& pIemRec->pNext)
7642	{
7643	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
7644	iemVerifyWriteRecord(pIemCpu, pIemRec);
7645	pIemRec = pIemRec->pNext;
7646	}
7647
7648	/* Do the compare. */
7649	if (pIemRec->enmEvent != pOtherRec->enmEvent)
7650	{
7651	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Type mismatches");
7652	break;
7653	}
7654	bool fEquals;
7655	switch (pIemRec->enmEvent)
7656	{
7657	case IEMVERIFYEVENT_IOPORT_READ:
7658	fEquals = pIemRec->u.IOPortRead.Port == pOtherRec->u.IOPortRead.Port
7659	&& pIemRec->u.IOPortRead.cbValue == pOtherRec->u.IOPortRead.cbValue;
7660	break;
7661	case IEMVERIFYEVENT_IOPORT_WRITE:
7662	fEquals = pIemRec->u.IOPortWrite.Port == pOtherRec->u.IOPortWrite.Port
7663	&& pIemRec->u.IOPortWrite.cbValue == pOtherRec->u.IOPortWrite.cbValue
7664	&& pIemRec->u.IOPortWrite.u32Value == pOtherRec->u.IOPortWrite.u32Value;
7665	break;
7666	case IEMVERIFYEVENT_RAM_READ:
7667	fEquals = pIemRec->u.RamRead.GCPhys == pOtherRec->u.RamRead.GCPhys
7668	&& pIemRec->u.RamRead.cb == pOtherRec->u.RamRead.cb;
7669	break;
7670	case IEMVERIFYEVENT_RAM_WRITE:
7671	fEquals = pIemRec->u.RamWrite.GCPhys == pOtherRec->u.RamWrite.GCPhys
7672	&& pIemRec->u.RamWrite.cb == pOtherRec->u.RamWrite.cb
7673	&& !memcmp(pIemRec->u.RamWrite.ab, pOtherRec->u.RamWrite.ab, pIemRec->u.RamWrite.cb);
7674	break;
7675	default:
7676	fEquals = false;
7677	break;
7678	}
7679	if (!fEquals)
7680	{
7681	iemVerifyAssertRecords(pIemCpu, pIemRec, pOtherRec, "Mismatch");
7682	break;
7683	}
7684
7685	/* advance */
7686	pIemRec = pIemRec->pNext;
7687	pOtherRec = pOtherRec->pNext;
7688	}
7689
7690	/* Ignore extra writes and reads. */
7691	while (pIemRec && IEMVERIFYEVENT_IS_RAM(pIemRec->enmEvent))
7692	{
7693	if (pIemRec->enmEvent == IEMVERIFYEVENT_RAM_WRITE)
7694	iemVerifyWriteRecord(pIemCpu, pIemRec);
7695	pIemRec = pIemRec->pNext;
7696	}
7697	if (pIemRec != NULL)
7698	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra IEM record!");
7699	else if (pOtherRec != NULL)
7700	iemVerifyAssertRecord(pIemCpu, pIemRec, "Extra Other record!");
7701	}
7702	pIemCpu->CTX_SUFF(pCtx) = pOrgCtx;
7703
7704	#if 0
7705	/*
7706	* HACK ALERT! You don't normally want to verify a whole boot sequence.
7707	*/
7708	if (pIemCpu->cInstructions == 1)
7709	RTLogFlags(NULL, "disabled");
7710	#endif
7711	}
7712
7713	#else /* !IEM_VERIFICATION_MODE \|\| !IN_RING3 */
7714
7715	/* stubs */
7716	static VBOXSTRICTRC iemVerifyFakeIOPortRead(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
7717	{
7718	NOREF(pIemCpu); NOREF(Port); NOREF(pu32Value); NOREF(cbValue);
7719	return VERR_INTERNAL_ERROR;
7720	}
7721
7722	static VBOXSTRICTRC iemVerifyFakeIOPortWrite(PIEMCPU pIemCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
7723	{
7724	NOREF(pIemCpu); NOREF(Port); NOREF(u32Value); NOREF(cbValue);
7725	return VERR_INTERNAL_ERROR;
7726	}
7727
7728	#endif /* !IEM_VERIFICATION_MODE \|\| !IN_RING3 */
7729
7730
7731	/**
7732	* Execute one instruction.
7733	*
7734	* @return Strict VBox status code.
7735	* @param pVCpu The current virtual CPU.
7736	*/
7737	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPU pVCpu)
7738	{
7739	PIEMCPU pIemCpu = &pVCpu->iem.s;
7740
7741	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7742	iemExecVerificationModeSetup(pIemCpu);
7743	#endif
7744	#ifdef LOG_ENABLED
7745	PCPUMCTX pCtx = pIemCpu->CTX_SUFF(pCtx);
7746	if (LogIs2Enabled())
7747	{
7748	char szInstr[256];
7749	uint32_t cbInstr = 0;
7750	DBGFR3DisasInstrEx(pVCpu->pVMR3, pVCpu->idCpu, 0, 0,
7751	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
7752	szInstr, sizeof(szInstr), &cbInstr);
7753
7754	Log3(("**** "
7755	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
7756	" eip=%08x esp=%08x ebp=%08x iopl=%d\n"
7757	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
7758	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
7759	" %s\n"
7760	,
7761	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
7762	pCtx->eip, pCtx->esp, pCtx->ebp, pCtx->eflags.Bits.u2IOPL,
7763	(RTSEL)pCtx->cs, (RTSEL)pCtx->ss, (RTSEL)pCtx->ds, (RTSEL)pCtx->es,
7764	(RTSEL)pCtx->fs, (RTSEL)pCtx->gs, pCtx->eflags.u,
7765	pCtx->fpu.FSW, pCtx->fpu.FCW, pCtx->fpu.FTW, pCtx->fpu.MXCSR, pCtx->fpu.MXCSR_MASK,
7766	szInstr));
7767
7768	if (LogIs3Enabled())
7769	DBGFR3Info(pVCpu->pVMR3, "cpumguest", "verbose", NULL);
7770	}
7771	else
7772	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n",
7773	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u));
7774	#endif
7775
7776	/*
7777	* Do the decoding and emulation.
7778	*/
7779	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu);
7780	if (rcStrict != VINF_SUCCESS)
7781	{
7782	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7783	iemExecVerificationModeCheck(pIemCpu);
7784	#endif
7785	return rcStrict;
7786	}
7787
7788	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
7789	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
7790	if (rcStrict == VINF_SUCCESS)
7791	pIemCpu->cInstructions++;
7792	//#ifdef DEBUG
7793	// AssertMsg(pIemCpu->offOpcode == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", pIemCpu->offOpcode, cbInstr));
7794	//#endif
7795
7796	/* Execute the next instruction as well if a cli, pop ss or
7797	mov ss, Gr has just completed successfully. */
7798	if ( rcStrict == VINF_SUCCESS
7799	&& VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
7800	&& EMGetInhibitInterruptsPC(pVCpu) == pIemCpu->CTX_SUFF(pCtx)->rip )
7801	{
7802	rcStrict = iemInitDecoderAndPrefetchOpcodes(pIemCpu);
7803	if (rcStrict == VINF_SUCCESS)
7804	{
7805	b; IEM_OPCODE_GET_NEXT_U8(&b);
7806	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
7807	if (rcStrict == VINF_SUCCESS)
7808	pIemCpu->cInstructions++;
7809	}
7810	EMSetInhibitInterruptsPC(pVCpu, UINT64_C(0x7777555533331111));
7811	}
7812
7813	#if defined(IEM_VERIFICATION_MODE) && defined(IN_RING3)
7814	/*
7815	* Assert some sanity.
7816	*/
7817	iemExecVerificationModeCheck(pIemCpu);
7818	#endif
7819	if (rcStrict != VINF_SUCCESS)
7820	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
7821	pCtx->cs, pCtx->rip, pCtx->ss, pCtx->rsp, pCtx->eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
7822	return rcStrict;
7823	}
7824
7825
7826	/**
7827	* Injects a trap, fault, abort, software interrupt or external interrupt.
7828	*
7829	* The parameter list matches TRPMQueryTrapAll pretty closely.
7830	*
7831	* @returns Strict VBox status code.
7832	* @param pVCpu The current virtual CPU.
7833	* @param u8TrapNo The trap number.
7834	* @param enmType What type is it (trap/fault/abort), software
7835	* interrupt or hardware interrupt.
7836	* @param uErrCode The error code if applicable.
7837	* @param uCr2 The CR2 value if applicable.
7838	*/
7839	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPU pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2)
7840	{
7841	iemInitDecoder(&pVCpu->iem.s);
7842
7843	uint32_t fFlags;
7844	switch (enmType)
7845	{
7846	case TRPM_HARDWARE_INT:
7847	LogFlow(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
7848	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
7849	uErrCode = uCr2 = 0;
7850	break;
7851
7852	case TRPM_SOFTWARE_INT:
7853	LogFlow(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
7854	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
7855	uErrCode = uCr2 = 0;
7856	break;
7857
7858	case TRPM_TRAP:
7859	LogFlow(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
7860	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
7861	if (u8TrapNo == X86_XCPT_PF)
7862	fFlags \|= IEM_XCPT_FLAGS_CR2;
7863	switch (u8TrapNo)
7864	{
7865	case X86_XCPT_DF:
7866	case X86_XCPT_TS:
7867	case X86_XCPT_NP:
7868	case X86_XCPT_SS:
7869	case X86_XCPT_PF:
7870	case X86_XCPT_AC:
7871	fFlags \|= IEM_XCPT_FLAGS_ERR;
7872	break;
7873	}
7874	break;
7875
7876	IEM_NOT_REACHED_DEFAULT_CASE_RET();
7877	}
7878
7879	return iemRaiseXcptOrInt(&pVCpu->iem.s, 0, u8TrapNo, fFlags, uErrCode, uCr2);
7880	}
7881

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 40256

Download in other formats: