IEMAll.cpp@ 103917

Last change on this file since 103917 was 103916, checked in by vboxsync, 9 months ago
VMM/IEM: Convert iemMemStoreDataU256NoAc()/iemMemStoreDataU256NoAcJmp() to use the memory RW template and implement native emitters for IEM_MC_FETCH_MEM_U256_NO_AC()/IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(), bugref:10614
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 443.5 KB

Line
1	/* $Id: IEMAll.cpp 103916 2024-03-19 13:11:09Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/** @page pg_iem IEM - Interpreted Execution Manager
30	*
31	* The interpreted exeuction manager (IEM) is for executing short guest code
32	* sequences that are causing too many exits / virtualization traps. It will
33	* also be used to interpret single instructions, thus replacing the selective
34	* interpreters in EM and IOM.
35	*
36	* Design goals:
37	* - Relatively small footprint, although we favour speed and correctness
38	* over size.
39	* - Reasonably fast.
40	* - Correctly handle lock prefixed instructions.
41	* - Complete instruction set - eventually.
42	* - Refactorable into a recompiler, maybe.
43	* - Replace EMInterpret*.
44	*
45	* Using the existing disassembler has been considered, however this is thought
46	* to conflict with speed as the disassembler chews things a bit too much while
47	* leaving us with a somewhat complicated state to interpret afterwards.
48	*
49	*
50	* The current code is very much work in progress. You've been warned!
51	*
52	*
53	* @section sec_iem_fpu_instr FPU Instructions
54	*
55	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
56	* same or equivalent instructions on the host FPU. To make life easy, we also
57	* let the FPU prioritize the unmasked exceptions for us. This however, only
58	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
59	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
60	* can trigger spurious FPU exceptions.
61	*
62	* The guest FPU state is not loaded into the host CPU and kept there till we
63	* leave IEM because the calling conventions have declared an all year open
64	* season on much of the FPU state. For instance an innocent looking call to
65	* memcpy might end up using a whole bunch of XMM or MM registers if the
66	* particular implementation finds it worthwhile.
67	*
68	*
69	* @section sec_iem_logging Logging
70	*
71	* The IEM code uses the \"IEM\" log group for the main logging. The different
72	* logging levels/flags are generally used for the following purposes:
73	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
74	* - Flow (LogFlow) : Basic enter/exit IEM state info.
75	* - Level 2 (Log2) : ?
76	* - Level 3 (Log3) : More detailed enter/exit IEM state info.
77	* - Level 4 (Log4) : Decoding mnemonics w/ EIP.
78	* - Level 5 (Log5) : Decoding details.
79	* - Level 6 (Log6) : Enables/disables the lockstep comparison with REM.
80	* - Level 7 (Log7) : iret++ execution logging.
81	* - Level 8 (Log8) :
82	* - Level 9 (Log9) :
83	* - Level 10 (Log10): TLBs.
84	* - Level 11 (Log11): Unmasked FPU exceptions.
85	*
86	* The \"IEM_MEM\" log group covers most of memory related details logging,
87	* except for errors and exceptions:
88	* - Level 1 (Log) : Reads.
89	* - Level 2 (Log2) : Read fallbacks.
90	* - Level 3 (Log3) : MemMap read.
91	* - Level 4 (Log4) : MemMap read fallbacks.
92	* - Level 5 (Log5) : Writes
93	* - Level 6 (Log6) : Write fallbacks.
94	* - Level 7 (Log7) : MemMap writes and read-writes.
95	* - Level 8 (Log8) : MemMap write and read-write fallbacks.
96	* - Level 9 (Log9) : Stack reads.
97	* - Level 10 (Log10): Stack read fallbacks.
98	* - Level 11 (Log11): Stack writes.
99	* - Level 12 (Log12): Stack write fallbacks.
100	* - Flow (LogFlow) :
101	*
102	* The SVM (AMD-V) and VMX (VT-x) code has the following assignments:
103	* - Level 1 (Log) : Errors and other major events.
104	* - Flow (LogFlow) : Misc flow stuff (cleanup?)
105	* - Level 2 (Log2) : VM exits.
106	*
107	* The syscall logging level assignments:
108	* - Level 1: DOS and BIOS.
109	* - Level 2: Windows 3.x
110	* - Level 3: Linux.
111	*/
112
113	/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
114	#ifdef _MSC_VER
115	# pragma warning(disable:4505)
116	#endif
117
118
119	/*********************************************************************************************************************************
120	* Header Files *
121	*********************************************************************************************************************************/
122	#define LOG_GROUP LOG_GROUP_IEM
123	#define VMCPU_INCL_CPUM_GST_CTX
124	#include <VBox/vmm/iem.h>
125	#include <VBox/vmm/cpum.h>
126	#include <VBox/vmm/apic.h>
127	#include <VBox/vmm/pdm.h>
128	#include <VBox/vmm/pgm.h>
129	#include <VBox/vmm/iom.h>
130	#include <VBox/vmm/em.h>
131	#include <VBox/vmm/hm.h>
132	#include <VBox/vmm/nem.h>
133	#include <VBox/vmm/gim.h>
134	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
135	# include <VBox/vmm/em.h>
136	# include <VBox/vmm/hm_svm.h>
137	#endif
138	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
139	# include <VBox/vmm/hmvmxinline.h>
140	#endif
141	#include <VBox/vmm/tm.h>
142	#include <VBox/vmm/dbgf.h>
143	#include <VBox/vmm/dbgftrace.h>
144	#include "IEMInternal.h"
145	#include <VBox/vmm/vmcc.h>
146	#include <VBox/log.h>
147	#include <VBox/err.h>
148	#include <VBox/param.h>
149	#include <VBox/dis.h>
150	#include <iprt/asm-math.h>
151	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
152	# include <iprt/asm-amd64-x86.h>
153	#elif defined(RT_ARCH_ARM64) \|\| defined(RT_ARCH_ARM32)
154	# include <iprt/asm-arm.h>
155	#endif
156	#include <iprt/assert.h>
157	#include <iprt/string.h>
158	#include <iprt/x86.h>
159
160	#include "IEMInline.h"
161
162
163	/*********************************************************************************************************************************
164	* Structures and Typedefs *
165	*********************************************************************************************************************************/
166	/**
167	* CPU exception classes.
168	*/
169	typedef enum IEMXCPTCLASS
170	{
171	IEMXCPTCLASS_BENIGN,
172	IEMXCPTCLASS_CONTRIBUTORY,
173	IEMXCPTCLASS_PAGE_FAULT,
174	IEMXCPTCLASS_DOUBLE_FAULT
175	} IEMXCPTCLASS;
176
177
178	/*********************************************************************************************************************************
179	* Global Variables *
180	*********************************************************************************************************************************/
181	#if defined(IEM_LOG_MEMORY_WRITES)
182	/** What IEM just wrote. */
183	uint8_t g_abIemWrote[256];
184	/** How much IEM just wrote. */
185	size_t g_cbIemWrote;
186	#endif
187
188
189	/*********************************************************************************************************************************
190	* Internal Functions *
191	*********************************************************************************************************************************/
192	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
193	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT;
194
195
196	/**
197	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags, slow code
198	* path.
199	*
200	* @returns IEM_F_BRK_PENDING_XXX or zero.
201	* @param pVCpu The cross context virtual CPU structure of the
202	* calling thread.
203	*
204	* @note Don't call directly, use iemCalcExecDbgFlags instead.
205	*/
206	uint32_t iemCalcExecDbgFlagsSlow(PVMCPUCC pVCpu)
207	{
208	uint32_t fExec = 0;
209
210	/*
211	* Process guest breakpoints.
212	*/
213	#define PROCESS_ONE_BP(a_fDr7, a_iBp) do { \
214	if (a_fDr7 & X86_DR7_L_G(a_iBp)) \
215	{ \
216	switch (X86_DR7_GET_RW(a_fDr7, a_iBp)) \
217	{ \
218	case X86_DR7_RW_EO: \
219	fExec \|= IEM_F_PENDING_BRK_INSTR; \
220	break; \
221	case X86_DR7_RW_WO: \
222	case X86_DR7_RW_RW: \
223	fExec \|= IEM_F_PENDING_BRK_DATA; \
224	break; \
225	case X86_DR7_RW_IO: \
226	fExec \|= IEM_F_PENDING_BRK_X86_IO; \
227	break; \
228	} \
229	} \
230	} while (0)
231
232	uint32_t const fGstDr7 = (uint32_t)pVCpu->cpum.GstCtx.dr[7];
233	if (fGstDr7 & X86_DR7_ENABLED_MASK)
234	{
235	PROCESS_ONE_BP(fGstDr7, 0);
236	PROCESS_ONE_BP(fGstDr7, 1);
237	PROCESS_ONE_BP(fGstDr7, 2);
238	PROCESS_ONE_BP(fGstDr7, 3);
239	}
240
241	/*
242	* Process hypervisor breakpoints.
243	*/
244	uint32_t const fHyperDr7 = DBGFBpGetDR7(pVCpu->CTX_SUFF(pVM));
245	if (fHyperDr7 & X86_DR7_ENABLED_MASK)
246	{
247	PROCESS_ONE_BP(fHyperDr7, 0);
248	PROCESS_ONE_BP(fHyperDr7, 1);
249	PROCESS_ONE_BP(fHyperDr7, 2);
250	PROCESS_ONE_BP(fHyperDr7, 3);
251	}
252
253	return fExec;
254	}
255
256
257	/**
258	* Initializes the decoder state.
259	*
260	* iemReInitDecoder is mostly a copy of this function.
261	*
262	* @param pVCpu The cross context virtual CPU structure of the
263	* calling thread.
264	* @param fExecOpts Optional execution flags:
265	* - IEM_F_BYPASS_HANDLERS
266	* - IEM_F_X86_DISREGARD_LOCK
267	*/
268	DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, uint32_t fExecOpts)
269	{
270	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
271	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
272	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
273	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
274	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
275	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
276	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
277	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
278	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
279	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
280
281	/* Execution state: */
282	uint32_t fExec;
283	pVCpu->iem.s.fExec = fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
284
285	/* Decoder state: */
286	pVCpu->iem.s.enmDefAddrMode = fExec & IEM_F_MODE_CPUMODE_MASK; /** @todo check if this is correct... */
287	pVCpu->iem.s.enmEffAddrMode = fExec & IEM_F_MODE_CPUMODE_MASK;
288	if ((fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
289	{
290	pVCpu->iem.s.enmDefOpSize = fExec & IEM_F_MODE_CPUMODE_MASK; /** @todo check if this is correct... */
291	pVCpu->iem.s.enmEffOpSize = fExec & IEM_F_MODE_CPUMODE_MASK;
292	}
293	else
294	{
295	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
296	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
297	}
298	pVCpu->iem.s.fPrefixes = 0;
299	pVCpu->iem.s.uRexReg = 0;
300	pVCpu->iem.s.uRexB = 0;
301	pVCpu->iem.s.uRexIndex = 0;
302	pVCpu->iem.s.idxPrefix = 0;
303	pVCpu->iem.s.uVex3rdReg = 0;
304	pVCpu->iem.s.uVexLength = 0;
305	pVCpu->iem.s.fEvexStuff = 0;
306	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
307	#ifdef IEM_WITH_CODE_TLB
308	pVCpu->iem.s.pbInstrBuf = NULL;
309	pVCpu->iem.s.offInstrNextByte = 0;
310	pVCpu->iem.s.offCurInstrStart = 0;
311	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
312	pVCpu->iem.s.offOpcode = 0;
313	# endif
314	# ifdef VBOX_STRICT
315	pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
316	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
317	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
318	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
319	# endif
320	#else
321	pVCpu->iem.s.offOpcode = 0;
322	pVCpu->iem.s.cbOpcode = 0;
323	#endif
324	pVCpu->iem.s.offModRm = 0;
325	pVCpu->iem.s.cActiveMappings = 0;
326	pVCpu->iem.s.iNextMapping = 0;
327	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
328
329	#ifdef DBGFTRACE_ENABLED
330	switch (IEM_GET_CPU_MODE(pVCpu))
331	{
332	case IEMMODE_64BIT:
333	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);
334	break;
335	case IEMMODE_32BIT:
336	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
337	break;
338	case IEMMODE_16BIT:
339	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
340	break;
341	}
342	#endif
343	}
344
345
346	/**
347	* Reinitializes the decoder state 2nd+ loop of IEMExecLots.
348	*
349	* This is mostly a copy of iemInitDecoder.
350	*
351	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
352	*/
353	DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
354	{
355	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
356	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
357	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
358	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
359	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
360	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
361	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
362	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
363	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
364
365	/* ASSUMES: Anyone changing CPU state affecting the fExec bits will update them! */
366	AssertMsg((pVCpu->iem.s.fExec & ~IEM_F_USER_OPTS) == iemCalcExecFlags(pVCpu),
367	("fExec=%#x iemCalcExecModeFlags=%#x\n", pVCpu->iem.s.fExec, iemCalcExecFlags(pVCpu)));
368
369	IEMMODE const enmMode = IEM_GET_CPU_MODE(pVCpu);
370	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
371	pVCpu->iem.s.enmEffAddrMode = enmMode;
372	if (enmMode != IEMMODE_64BIT)
373	{
374	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
375	pVCpu->iem.s.enmEffOpSize = enmMode;
376	}
377	else
378	{
379	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
380	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
381	}
382	pVCpu->iem.s.fPrefixes = 0;
383	pVCpu->iem.s.uRexReg = 0;
384	pVCpu->iem.s.uRexB = 0;
385	pVCpu->iem.s.uRexIndex = 0;
386	pVCpu->iem.s.idxPrefix = 0;
387	pVCpu->iem.s.uVex3rdReg = 0;
388	pVCpu->iem.s.uVexLength = 0;
389	pVCpu->iem.s.fEvexStuff = 0;
390	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
391	#ifdef IEM_WITH_CODE_TLB
392	if (pVCpu->iem.s.pbInstrBuf)
393	{
394	uint64_t off = (enmMode == IEMMODE_64BIT
395	? pVCpu->cpum.GstCtx.rip
396	: pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
397	- pVCpu->iem.s.uInstrBufPc;
398	if (off < pVCpu->iem.s.cbInstrBufTotal)
399	{
400	pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
401	pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
402	if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
403	pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
404	else
405	pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
406	}
407	else
408	{
409	pVCpu->iem.s.pbInstrBuf = NULL;
410	pVCpu->iem.s.offInstrNextByte = 0;
411	pVCpu->iem.s.offCurInstrStart = 0;
412	pVCpu->iem.s.cbInstrBuf = 0;
413	pVCpu->iem.s.cbInstrBufTotal = 0;
414	pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
415	}
416	}
417	else
418	{
419	pVCpu->iem.s.offInstrNextByte = 0;
420	pVCpu->iem.s.offCurInstrStart = 0;
421	pVCpu->iem.s.cbInstrBuf = 0;
422	pVCpu->iem.s.cbInstrBufTotal = 0;
423	# ifdef VBOX_STRICT
424	pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
425	# endif
426	}
427	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
428	pVCpu->iem.s.offOpcode = 0;
429	# endif
430	#else /* !IEM_WITH_CODE_TLB */
431	pVCpu->iem.s.cbOpcode = 0;
432	pVCpu->iem.s.offOpcode = 0;
433	#endif /* !IEM_WITH_CODE_TLB */
434	pVCpu->iem.s.offModRm = 0;
435	Assert(pVCpu->iem.s.cActiveMappings == 0);
436	pVCpu->iem.s.iNextMapping = 0;
437	Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
438	Assert(!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS));
439
440	#ifdef DBGFTRACE_ENABLED
441	switch (enmMode)
442	{
443	case IEMMODE_64BIT:
444	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);
445	break;
446	case IEMMODE_32BIT:
447	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
448	break;
449	case IEMMODE_16BIT:
450	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
451	break;
452	}
453	#endif
454	}
455
456
457
458	/**
459	* Prefetch opcodes the first time when starting executing.
460	*
461	* @returns Strict VBox status code.
462	* @param pVCpu The cross context virtual CPU structure of the
463	* calling thread.
464	* @param fExecOpts Optional execution flags:
465	* - IEM_F_BYPASS_HANDLERS
466	* - IEM_F_X86_DISREGARD_LOCK
467	*/
468	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
469	{
470	iemInitDecoder(pVCpu, fExecOpts);
471
472	#ifndef IEM_WITH_CODE_TLB
473	/*
474	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
475	*
476	* First translate CS:rIP to a physical address.
477	*
478	* Note! The iemOpcodeFetchMoreBytes code depends on this here code to fetch
479	* all relevant bytes from the first page, as it ASSUMES it's only ever
480	* called for dealing with CS.LIM, page crossing and instructions that
481	* are too long.
482	*/
483	uint32_t cbToTryRead;
484	RTGCPTR GCPtrPC;
485	if (IEM_IS_64BIT_CODE(pVCpu))
486	{
487	cbToTryRead = GUEST_PAGE_SIZE;
488	GCPtrPC = pVCpu->cpum.GstCtx.rip;
489	if (IEM_IS_CANONICAL(GCPtrPC))
490	cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
491	else
492	return iemRaiseGeneralProtectionFault0(pVCpu);
493	}
494	else
495	{
496	uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
497	AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| IEM_IS_32BIT_CODE(pVCpu), ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
498	if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
499	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
500	else
501	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
502	if (cbToTryRead) { /* likely */ }
503	else /* overflowed */
504	{
505	Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
506	cbToTryRead = UINT32_MAX;
507	}
508	GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
509	Assert(GCPtrPC <= UINT32_MAX);
510	}
511
512	PGMPTWALK Walk;
513	int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
514	if (RT_SUCCESS(rc))
515	Assert(Walk.fSucceeded); /* probable. */
516	else
517	{
518	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
519	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
520	if (Walk.fFailed & PGM_WALKFAIL_EPT)
521	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
522	# endif
523	return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, rc);
524	}
525	if ((Walk.fEffective & X86_PTE_US) \|\| IEM_GET_CPL(pVCpu) != 3) { /* likely */ }
526	else
527	{
528	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
529	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
530	if (Walk.fFailed & PGM_WALKFAIL_EPT)
531	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
532	# endif
533	return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
534	}
535	if (!(Walk.fEffective & X86_PTE_PAE_NX) \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
536	else
537	{
538	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
539	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
540	if (Walk.fFailed & PGM_WALKFAIL_EPT)
541	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
542	# endif
543	return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
544	}
545	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
546	/** @todo Check reserved bits and such stuff. PGM is better at doing
547	* that, so do it when implementing the guest virtual address
548	* TLB... */
549
550	/*
551	* Read the bytes at this address.
552	*/
553	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
554	if (cbToTryRead > cbLeftOnPage)
555	cbToTryRead = cbLeftOnPage;
556	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
557	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
558
559	if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
560	{
561	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
562	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
563	{ /* likely */ }
564	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
565	{
566	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
567	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
568	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
569	}
570	else
571	{
572	Log((RT_SUCCESS(rcStrict)
573	? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
574	: "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
575	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
576	return rcStrict;
577	}
578	}
579	else
580	{
581	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
582	if (RT_SUCCESS(rc))
583	{ /* likely */ }
584	else
585	{
586	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
587	GCPtrPC, GCPhys, rc, cbToTryRead));
588	return rc;
589	}
590	}
591	pVCpu->iem.s.cbOpcode = cbToTryRead;
592	#endif /* !IEM_WITH_CODE_TLB */
593	return VINF_SUCCESS;
594	}
595
596
597	/**
598	* Invalidates the IEM TLBs.
599	*
600	* This is called internally as well as by PGM when moving GC mappings.
601	*
602	* @param pVCpu The cross context virtual CPU structure of the calling
603	* thread.
604	*/
605	VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)
606	{
607	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
608	Log10(("IEMTlbInvalidateAll\n"));
609	# ifdef IEM_WITH_CODE_TLB
610	pVCpu->iem.s.cbInstrBufTotal = 0;
611	pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
612	if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
613	{ /* very likely */ }
614	else
615	{
616	pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
617	unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
618	while (i-- > 0)
619	pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
620	}
621	# endif
622
623	# ifdef IEM_WITH_DATA_TLB
624	pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
625	if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
626	{ /* very likely */ }
627	else
628	{
629	pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
630	unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
631	while (i-- > 0)
632	pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
633	}
634	# endif
635	#else
636	RT_NOREF(pVCpu);
637	#endif
638	}
639
640
641	/**
642	* Invalidates a page in the TLBs.
643	*
644	* @param pVCpu The cross context virtual CPU structure of the calling
645	* thread.
646	* @param GCPtr The address of the page to invalidate
647	* @thread EMT(pVCpu)
648	*/
649	VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
650	{
651	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
652	Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));
653	GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);
654	Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));
655	uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr);
656
657	# ifdef IEM_WITH_CODE_TLB
658	if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.CodeTlb.uTlbRevision))
659	{
660	pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
661	if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc))
662	pVCpu->iem.s.cbInstrBufTotal = 0;
663	}
664	# endif
665
666	# ifdef IEM_WITH_DATA_TLB
667	if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.DataTlb.uTlbRevision))
668	pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
669	# endif
670	#else
671	NOREF(pVCpu); NOREF(GCPtr);
672	#endif
673	}
674
675
676	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
677	/**
678	* Invalid both TLBs slow fashion following a rollover.
679	*
680	* Worker for IEMTlbInvalidateAllPhysical,
681	* IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,
682	* iemMemMapJmp and others.
683	*
684	* @thread EMT(pVCpu)
685	*/
686	static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu)
687	{
688	Log10(("IEMTlbInvalidateAllPhysicalSlow\n"));
689	ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
690	ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
691
692	unsigned i;
693	# ifdef IEM_WITH_CODE_TLB
694	i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
695	while (i-- > 0)
696	{
697	pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
698	pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
699	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
700	}
701	# endif
702	# ifdef IEM_WITH_DATA_TLB
703	i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
704	while (i-- > 0)
705	{
706	pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
707	pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
708	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
709	}
710	# endif
711
712	}
713	#endif
714
715
716	/**
717	* Invalidates the host physical aspects of the IEM TLBs.
718	*
719	* This is called internally as well as by PGM when moving GC mappings.
720	*
721	* @param pVCpu The cross context virtual CPU structure of the calling
722	* thread.
723	* @note Currently not used.
724	*/
725	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
726	{
727	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
728	/* Note! This probably won't end up looking exactly like this, but it give an idea... */
729	Log10(("IEMTlbInvalidateAllPhysical\n"));
730
731	# ifdef IEM_WITH_CODE_TLB
732	pVCpu->iem.s.cbInstrBufTotal = 0;
733	# endif
734	uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
735	if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))
736	{
737	pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
738	pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
739	}
740	else
741	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
742	#else
743	NOREF(pVCpu);
744	#endif
745	}
746
747
748	/**
749	* Invalidates the host physical aspects of the IEM TLBs.
750	*
751	* This is called internally as well as by PGM when moving GC mappings.
752	*
753	* @param pVM The cross context VM structure.
754	* @param idCpuCaller The ID of the calling EMT if available to the caller,
755	* otherwise NIL_VMCPUID.
756	* @param enmReason The reason we're called.
757	*
758	* @remarks Caller holds the PGM lock.
759	*/
760	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller, IEMTLBPHYSFLUSHREASON enmReason)
761	{
762	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
763	PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);
764	if (pVCpuCaller)
765	VMCPU_ASSERT_EMT(pVCpuCaller);
766	Log10(("IEMTlbInvalidateAllPhysicalAllCpus: %d\n", enmReason)); RT_NOREF(enmReason);
767
768	VMCC_FOR_EACH_VMCPU(pVM)
769	{
770	# ifdef IEM_WITH_CODE_TLB
771	if (pVCpuCaller == pVCpu)
772	pVCpu->iem.s.cbInstrBufTotal = 0;
773	# endif
774
775	uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);
776	uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;
777	if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))
778	{ /* likely */}
779	else if (pVCpuCaller != pVCpu)
780	uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;
781	else
782	{
783	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
784	continue;
785	}
786	ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
787	ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
788	}
789	VMCC_FOR_EACH_VMCPU_END(pVM);
790
791	#else
792	RT_NOREF(pVM, idCpuCaller, enmReason);
793	#endif
794	}
795
796
797	/**
798	* Flushes the prefetch buffer, light version.
799	*/
800	void iemOpcodeFlushLight(PVMCPUCC pVCpu, uint8_t cbInstr)
801	{
802	#ifndef IEM_WITH_CODE_TLB
803	pVCpu->iem.s.cbOpcode = cbInstr;
804	#else
805	RT_NOREF(pVCpu, cbInstr);
806	#endif
807	}
808
809
810	/**
811	* Flushes the prefetch buffer, heavy version.
812	*/
813	void iemOpcodeFlushHeavy(PVMCPUCC pVCpu, uint8_t cbInstr)
814	{
815	#ifndef IEM_WITH_CODE_TLB
816	pVCpu->iem.s.cbOpcode = cbInstr; /* Note! SVM and VT-x may set this to zero on exit, rather than the instruction length. */
817	#elif 1
818	pVCpu->iem.s.cbInstrBufTotal = 0;
819	RT_NOREF(cbInstr);
820	#else
821	RT_NOREF(pVCpu, cbInstr);
822	#endif
823	}
824
825
826
827	#ifdef IEM_WITH_CODE_TLB
828
829	/**
830	* Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
831	* failure and jumps.
832	*
833	* We end up here for a number of reasons:
834	* - pbInstrBuf isn't yet initialized.
835	* - Advancing beyond the buffer boundrary (e.g. cross page).
836	* - Advancing beyond the CS segment limit.
837	* - Fetching from non-mappable page (e.g. MMIO).
838	* - TLB loading in the recompiler (@a pvDst = NULL, @a cbDst = 0).
839	*
840	* @param pVCpu The cross context virtual CPU structure of the
841	* calling thread.
842	* @param pvDst Where to return the bytes.
843	* @param cbDst Number of bytes to read. A value of zero is
844	* allowed for initializing pbInstrBuf (the
845	* recompiler does this). In this case it is best
846	* to set pbInstrBuf to NULL prior to the call.
847	*/
848	void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) IEM_NOEXCEPT_MAY_LONGJMP
849	{
850	# ifdef IN_RING3
851	for (;;)
852	{
853	Assert(cbDst <= 8);
854	uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
855
856	/*
857	* We might have a partial buffer match, deal with that first to make the
858	* rest simpler. This is the first part of the cross page/buffer case.
859	*/
860	uint8_t const * const pbInstrBuf = pVCpu->iem.s.pbInstrBuf;
861	if (pbInstrBuf != NULL)
862	{
863	Assert(cbDst != 0); /* pbInstrBuf shall be NULL in case of a TLB load */
864	uint32_t const cbInstrBuf = pVCpu->iem.s.cbInstrBuf;
865	if (offBuf < cbInstrBuf)
866	{
867	Assert(offBuf + cbDst > cbInstrBuf);
868	uint32_t const cbCopy = cbInstrBuf - offBuf;
869	memcpy(pvDst, &pbInstrBuf[offBuf], cbCopy);
870
871	cbDst -= cbCopy;
872	pvDst = (uint8_t *)pvDst + cbCopy;
873	offBuf += cbCopy;
874	}
875	}
876
877	/*
878	* Check segment limit, figuring how much we're allowed to access at this point.
879	*
880	* We will fault immediately if RIP is past the segment limit / in non-canonical
881	* territory. If we do continue, there are one or more bytes to read before we
882	* end up in trouble and we need to do that first before faulting.
883	*/
884	RTGCPTR GCPtrFirst;
885	uint32_t cbMaxRead;
886	if (IEM_IS_64BIT_CODE(pVCpu))
887	{
888	GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
889	if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
890	{ /* likely */ }
891	else
892	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
893	cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
894	}
895	else
896	{
897	GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
898	/* Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) \|\| IEM_IS_32BIT_CODE(pVCpu)); - this is allowed */
899	if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
900	{ /* likely */ }
901	else /** @todo For CPUs older than the 386, we should not necessarily generate \#GP here but wrap around! */
902	iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
903	cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
904	if (cbMaxRead != 0)
905	{ /* likely */ }
906	else
907	{
908	/* Overflowed because address is 0 and limit is max. */
909	Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
910	cbMaxRead = X86_PAGE_SIZE;
911	}
912	GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
913	uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
914	if (cbMaxRead2 < cbMaxRead)
915	cbMaxRead = cbMaxRead2;
916	/** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
917	}
918
919	/*
920	* Get the TLB entry for this piece of code.
921	*/
922	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.CodeTlb, GCPtrFirst);
923	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag);
924	if (pTlbe->uTag == uTag)
925	{
926	/* likely when executing lots of code, otherwise unlikely */
927	# ifdef VBOX_WITH_STATISTICS
928	pVCpu->iem.s.CodeTlb.cTlbHits++;
929	# endif
930	}
931	else
932	{
933	pVCpu->iem.s.CodeTlb.cTlbMisses++;
934	PGMPTWALK Walk;
935	int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
936	if (RT_FAILURE(rc))
937	{
938	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
939	/** @todo Nested VMX: Need to handle EPT violation/misconfig here? */
940	Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
941	#endif
942	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
943	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, rc);
944	}
945
946	AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
947	Assert(Walk.fSucceeded);
948	pTlbe->uTag = uTag;
949	pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A))
950	\| (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
951	pTlbe->GCPhys = Walk.GCPhys;
952	pTlbe->pbMappingR3 = NULL;
953	}
954
955	/*
956	* Check TLB page table level access flags.
957	*/
958	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_EXEC))
959	{
960	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && IEM_GET_CPL(pVCpu) == 3)
961	{
962	Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
963	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
964	}
965	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
966	{
967	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
968	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
969	}
970	}
971
972	/*
973	* Set the accessed flags.
974	* ASSUMES this is set when the address is translated rather than on commit...
975	*/
976	/** @todo testcase: check when the A bit are actually set by the CPU for code. */
977	if (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_ACCESSED)
978	{
979	int rc2 = PGMGstModifyPage(pVCpu, GCPtrFirst, 1, X86_PTE_A, ~(uint64_t)X86_PTE_A);
980	AssertRC(rc2);
981	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
982	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
983	pTlbe->fFlagsAndPhysRev &= ~IEMTLBE_F_PT_NO_ACCESSED;
984	}
985
986	/*
987	* Look up the physical page info if necessary.
988	*/
989	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
990	{ /* not necessary */ }
991	else
992	{
993	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
994	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
995	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
996	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
997	AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
998	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
999	{ /* likely */ }
1000	else
1001	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
1002	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
1003	\| IEMTLBE_F_NO_MAPPINGR3
1004	\| IEMTLBE_F_PG_NO_READ
1005	\| IEMTLBE_F_PG_NO_WRITE
1006	\| IEMTLBE_F_PG_UNASSIGNED
1007	\| IEMTLBE_F_PG_CODE_PAGE);
1008	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
1009	&pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
1010	AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc));
1011	}
1012
1013	# if defined(IN_RING3) \|\| defined(IN_RING0) /** @todo fixme */
1014	/*
1015	* Try do a direct read using the pbMappingR3 pointer.
1016	*/
1017	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ))
1018	== pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1019	{
1020	uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
1021	pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
1022	if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
1023	{
1024	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
1025	pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
1026	}
1027	else
1028	{
1029	uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
1030	if (cbInstr + (uint32_t)cbDst <= 15)
1031	{
1032	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
1033	pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
1034	}
1035	else
1036	{
1037	Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n",
1038	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst));
1039	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
1040	}
1041	}
1042	if (cbDst <= cbMaxRead)
1043	{
1044	pVCpu->iem.s.fTbCrossedPage \|= offPg == 0 \|\| pVCpu->iem.s.fTbBranched != 0; /** @todo Spurious load effect on branch handling? */
1045	pVCpu->iem.s.GCPhysInstrBufPrev = pVCpu->iem.s.GCPhysInstrBuf;
1046
1047	pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
1048	pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
1049	pVCpu->iem.s.GCPhysInstrBuf = pTlbe->GCPhys;
1050	pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
1051	if (cbDst > 0) /* To make ASAN happy in the TLB load case. */
1052	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
1053	else
1054	Assert(!pvDst);
1055	return;
1056	}
1057	pVCpu->iem.s.pbInstrBuf = NULL;
1058
1059	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
1060	pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
1061	}
1062	# else
1063	# error "refactor as needed"
1064	/*
1065	* If there is no special read handling, so we can read a bit more and
1066	* put it in the prefetch buffer.
1067	*/
1068	if ( cbDst < cbMaxRead
1069	&& (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1070	{
1071	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
1072	&pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
1073	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1074	{ /* likely */ }
1075	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1076	{
1077	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1078	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1079	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1080	AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICRC_VAL(rcStrict)));
1081	}
1082	else
1083	{
1084	Log((RT_SUCCESS(rcStrict)
1085	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1086	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1087	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1088	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
1089	}
1090	}
1091	# endif
1092	/*
1093	* Special read handling, so only read exactly what's needed.
1094	* This is a highly unlikely scenario.
1095	*/
1096	else
1097	{
1098	pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
1099
1100	/* Check instruction length. */
1101	uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
1102	if (RT_LIKELY(cbInstr + cbDst <= 15))
1103	{ /* likely */ }
1104	else
1105	{
1106	Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0) [slow]\n",
1107	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst));
1108	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
1109	}
1110
1111	/* Do the reading. */
1112	uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
1113	if (cbToRead > 0)
1114	{
1115	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
1116	pvDst, cbToRead, PGMACCESSORIGIN_IEM);
1117	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1118	{ /* likely */ }
1119	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1120	{
1121	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1122	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1123	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1124	AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)));
1125	}
1126	else
1127	{
1128	Log((RT_SUCCESS(rcStrict)
1129	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1130	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1131	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1132	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
1133	}
1134	}
1135
1136	/* Update the state and probably return. */
1137	uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
1138	pVCpu->iem.s.fTbCrossedPage \|= offPg == 0 \|\| pVCpu->iem.s.fTbBranched != 0;
1139	pVCpu->iem.s.GCPhysInstrBufPrev = pVCpu->iem.s.GCPhysInstrBuf;
1140
1141	pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
1142	pVCpu->iem.s.offInstrNextByte = offPg + cbInstr + cbToRead;
1143	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead + cbInstr) - cbToRead - cbInstr;
1144	pVCpu->iem.s.cbInstrBufTotal = X86_PAGE_SIZE; /** @todo ??? */
1145	pVCpu->iem.s.GCPhysInstrBuf = pTlbe->GCPhys;
1146	pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
1147	pVCpu->iem.s.pbInstrBuf = NULL;
1148	if (cbToRead == cbDst)
1149	return;
1150	Assert(cbToRead == cbMaxRead);
1151	}
1152
1153	/*
1154	* More to read, loop.
1155	*/
1156	cbDst -= cbMaxRead;
1157	pvDst = (uint8_t *)pvDst + cbMaxRead;
1158	}
1159	# else /* !IN_RING3 */
1160	RT_NOREF(pvDst, cbDst);
1161	if (pvDst \|\| cbDst)
1162	IEM_DO_LONGJMP(pVCpu, VERR_INTERNAL_ERROR);
1163	# endif /* !IN_RING3 */
1164	}
1165
1166	#else /* !IEM_WITH_CODE_TLB */
1167
1168	/**
1169	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
1170	* exception if it fails.
1171	*
1172	* @returns Strict VBox status code.
1173	* @param pVCpu The cross context virtual CPU structure of the
1174	* calling thread.
1175	* @param cbMin The minimum number of bytes relative offOpcode
1176	* that must be read.
1177	*/
1178	VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
1179	{
1180	/*
1181	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1182	*
1183	* First translate CS:rIP to a physical address.
1184	*/
1185	uint8_t const cbOpcode = pVCpu->iem.s.cbOpcode;
1186	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1187	uint8_t const cbLeft = cbOpcode - offOpcode;
1188	Assert(cbLeft < cbMin);
1189	Assert(cbOpcode <= sizeof(pVCpu->iem.s.abOpcode));
1190
1191	uint32_t cbToTryRead;
1192	RTGCPTR GCPtrNext;
1193	if (IEM_IS_64BIT_CODE(pVCpu))
1194	{
1195	GCPtrNext = pVCpu->cpum.GstCtx.rip + cbOpcode;
1196	if (!IEM_IS_CANONICAL(GCPtrNext))
1197	return iemRaiseGeneralProtectionFault0(pVCpu);
1198	cbToTryRead = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1199	}
1200	else
1201	{
1202	uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
1203	/* Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| IEM_IS_32BIT_CODE(pVCpu)); - this is allowed */
1204	GCPtrNext32 += cbOpcode;
1205	if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1206	/** @todo For CPUs older than the 386, we should not generate \#GP here but wrap around! */
1207	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1208	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1209	if (!cbToTryRead) /* overflowed */
1210	{
1211	Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1212	cbToTryRead = UINT32_MAX;
1213	/** @todo check out wrapping around the code segment. */
1214	}
1215	if (cbToTryRead < cbMin - cbLeft)
1216	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1217	GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1218
1219	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1220	if (cbToTryRead > cbLeftOnPage)
1221	cbToTryRead = cbLeftOnPage;
1222	}
1223
1224	/* Restrict to opcode buffer space.
1225
1226	We're making ASSUMPTIONS here based on work done previously in
1227	iemInitDecoderAndPrefetchOpcodes, where bytes from the first page will
1228	be fetched in case of an instruction crossing two pages. */
1229	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - cbOpcode)
1230	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - cbOpcode;
1231	if (RT_LIKELY(cbToTryRead + cbLeft >= cbMin))
1232	{ /* likely */ }
1233	else
1234	{
1235	Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n",
1236	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, offOpcode, cbMin));
1237	return iemRaiseGeneralProtectionFault0(pVCpu);
1238	}
1239
1240	PGMPTWALK Walk;
1241	int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
1242	if (RT_FAILURE(rc))
1243	{
1244	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1245	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1246	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1247	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1248	#endif
1249	return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, rc);
1250	}
1251	if (!(Walk.fEffective & X86_PTE_US) && IEM_GET_CPL(pVCpu) == 3)
1252	{
1253	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1254	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1255	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1256	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1257	#endif
1258	return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1259	}
1260	if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1261	{
1262	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1263	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1264	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1265	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1266	#endif
1267	return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1268	}
1269	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1270	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, cbOpcode));
1271	/** @todo Check reserved bits and such stuff. PGM is better at doing
1272	* that, so do it when implementing the guest virtual address
1273	* TLB... */
1274
1275	/*
1276	* Read the bytes at this address.
1277	*
1278	* We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1279	* and since PATM should only patch the start of an instruction there
1280	* should be no need to check again here.
1281	*/
1282	if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
1283	{
1284	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[cbOpcode],
1285	cbToTryRead, PGMACCESSORIGIN_IEM);
1286	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1287	{ /* likely */ }
1288	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1289	{
1290	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1291	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1292	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1293	}
1294	else
1295	{
1296	Log((RT_SUCCESS(rcStrict)
1297	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1298	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1299	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1300	return rcStrict;
1301	}
1302	}
1303	else
1304	{
1305	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[cbOpcode], GCPhys, cbToTryRead);
1306	if (RT_SUCCESS(rc))
1307	{ /* likely */ }
1308	else
1309	{
1310	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
1311	return rc;
1312	}
1313	}
1314	pVCpu->iem.s.cbOpcode = cbOpcode + cbToTryRead;
1315	Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
1316
1317	return VINF_SUCCESS;
1318	}
1319
1320	#endif /* !IEM_WITH_CODE_TLB */
1321	#ifndef IEM_WITH_SETJMP
1322
1323	/**
1324	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
1325	*
1326	* @returns Strict VBox status code.
1327	* @param pVCpu The cross context virtual CPU structure of the
1328	* calling thread.
1329	* @param pb Where to return the opcode byte.
1330	*/
1331	VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
1332	{
1333	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1334	if (rcStrict == VINF_SUCCESS)
1335	{
1336	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1337	*pb = pVCpu->iem.s.abOpcode[offOpcode];
1338	pVCpu->iem.s.offOpcode = offOpcode + 1;
1339	}
1340	else
1341	*pb = 0;
1342	return rcStrict;
1343	}
1344
1345	#else /* IEM_WITH_SETJMP */
1346
1347	/**
1348	* Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
1349	*
1350	* @returns The opcode byte.
1351	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1352	*/
1353	uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1354	{
1355	# ifdef IEM_WITH_CODE_TLB
1356	uint8_t u8;
1357	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
1358	return u8;
1359	# else
1360	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1361	if (rcStrict == VINF_SUCCESS)
1362	return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
1363	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
1364	# endif
1365	}
1366
1367	#endif /* IEM_WITH_SETJMP */
1368
1369	#ifndef IEM_WITH_SETJMP
1370
1371	/**
1372	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
1373	*
1374	* @returns Strict VBox status code.
1375	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1376	* @param pu16 Where to return the opcode dword.
1377	*/
1378	VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1379	{
1380	uint8_t u8;
1381	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1382	if (rcStrict == VINF_SUCCESS)
1383	*pu16 = (int8_t)u8;
1384	return rcStrict;
1385	}
1386
1387
1388	/**
1389	* Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
1390	*
1391	* @returns Strict VBox status code.
1392	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1393	* @param pu32 Where to return the opcode dword.
1394	*/
1395	VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1396	{
1397	uint8_t u8;
1398	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1399	if (rcStrict == VINF_SUCCESS)
1400	*pu32 = (int8_t)u8;
1401	return rcStrict;
1402	}
1403
1404
1405	/**
1406	* Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
1407	*
1408	* @returns Strict VBox status code.
1409	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1410	* @param pu64 Where to return the opcode qword.
1411	*/
1412	VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1413	{
1414	uint8_t u8;
1415	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1416	if (rcStrict == VINF_SUCCESS)
1417	*pu64 = (int8_t)u8;
1418	return rcStrict;
1419	}
1420
1421	#endif /* !IEM_WITH_SETJMP */
1422
1423
1424	#ifndef IEM_WITH_SETJMP
1425
1426	/**
1427	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
1428	*
1429	* @returns Strict VBox status code.
1430	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1431	* @param pu16 Where to return the opcode word.
1432	*/
1433	VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1434	{
1435	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1436	if (rcStrict == VINF_SUCCESS)
1437	{
1438	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1439	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1440	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1441	# else
1442	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1443	# endif
1444	pVCpu->iem.s.offOpcode = offOpcode + 2;
1445	}
1446	else
1447	*pu16 = 0;
1448	return rcStrict;
1449	}
1450
1451	#else /* IEM_WITH_SETJMP */
1452
1453	/**
1454	* Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
1455	*
1456	* @returns The opcode word.
1457	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1458	*/
1459	uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1460	{
1461	# ifdef IEM_WITH_CODE_TLB
1462	uint16_t u16;
1463	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
1464	return u16;
1465	# else
1466	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1467	if (rcStrict == VINF_SUCCESS)
1468	{
1469	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1470	pVCpu->iem.s.offOpcode += 2;
1471	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1472	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1473	# else
1474	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1475	# endif
1476	}
1477	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
1478	# endif
1479	}
1480
1481	#endif /* IEM_WITH_SETJMP */
1482
1483	#ifndef IEM_WITH_SETJMP
1484
1485	/**
1486	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1487	*
1488	* @returns Strict VBox status code.
1489	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1490	* @param pu32 Where to return the opcode double word.
1491	*/
1492	VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1493	{
1494	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1495	if (rcStrict == VINF_SUCCESS)
1496	{
1497	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1498	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1499	pVCpu->iem.s.offOpcode = offOpcode + 2;
1500	}
1501	else
1502	*pu32 = 0;
1503	return rcStrict;
1504	}
1505
1506
1507	/**
1508	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1509	*
1510	* @returns Strict VBox status code.
1511	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1512	* @param pu64 Where to return the opcode quad word.
1513	*/
1514	VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1515	{
1516	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1517	if (rcStrict == VINF_SUCCESS)
1518	{
1519	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1520	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1521	pVCpu->iem.s.offOpcode = offOpcode + 2;
1522	}
1523	else
1524	*pu64 = 0;
1525	return rcStrict;
1526	}
1527
1528	#endif /* !IEM_WITH_SETJMP */
1529
1530	#ifndef IEM_WITH_SETJMP
1531
1532	/**
1533	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1534	*
1535	* @returns Strict VBox status code.
1536	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1537	* @param pu32 Where to return the opcode dword.
1538	*/
1539	VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1540	{
1541	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1542	if (rcStrict == VINF_SUCCESS)
1543	{
1544	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1545	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1546	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1547	# else
1548	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1549	pVCpu->iem.s.abOpcode[offOpcode + 1],
1550	pVCpu->iem.s.abOpcode[offOpcode + 2],
1551	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1552	# endif
1553	pVCpu->iem.s.offOpcode = offOpcode + 4;
1554	}
1555	else
1556	*pu32 = 0;
1557	return rcStrict;
1558	}
1559
1560	#else /* IEM_WITH_SETJMP */
1561
1562	/**
1563	* Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
1564	*
1565	* @returns The opcode dword.
1566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1567	*/
1568	uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1569	{
1570	# ifdef IEM_WITH_CODE_TLB
1571	uint32_t u32;
1572	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
1573	return u32;
1574	# else
1575	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1576	if (rcStrict == VINF_SUCCESS)
1577	{
1578	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1579	pVCpu->iem.s.offOpcode = offOpcode + 4;
1580	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1581	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1582	# else
1583	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1584	pVCpu->iem.s.abOpcode[offOpcode + 1],
1585	pVCpu->iem.s.abOpcode[offOpcode + 2],
1586	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1587	# endif
1588	}
1589	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
1590	# endif
1591	}
1592
1593	#endif /* IEM_WITH_SETJMP */
1594
1595	#ifndef IEM_WITH_SETJMP
1596
1597	/**
1598	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1599	*
1600	* @returns Strict VBox status code.
1601	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1602	* @param pu64 Where to return the opcode dword.
1603	*/
1604	VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1605	{
1606	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1607	if (rcStrict == VINF_SUCCESS)
1608	{
1609	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1610	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1611	pVCpu->iem.s.abOpcode[offOpcode + 1],
1612	pVCpu->iem.s.abOpcode[offOpcode + 2],
1613	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1614	pVCpu->iem.s.offOpcode = offOpcode + 4;
1615	}
1616	else
1617	*pu64 = 0;
1618	return rcStrict;
1619	}
1620
1621
1622	/**
1623	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1624	*
1625	* @returns Strict VBox status code.
1626	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1627	* @param pu64 Where to return the opcode qword.
1628	*/
1629	VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1630	{
1631	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1632	if (rcStrict == VINF_SUCCESS)
1633	{
1634	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1635	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1636	pVCpu->iem.s.abOpcode[offOpcode + 1],
1637	pVCpu->iem.s.abOpcode[offOpcode + 2],
1638	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1639	pVCpu->iem.s.offOpcode = offOpcode + 4;
1640	}
1641	else
1642	*pu64 = 0;
1643	return rcStrict;
1644	}
1645
1646	#endif /* !IEM_WITH_SETJMP */
1647
1648	#ifndef IEM_WITH_SETJMP
1649
1650	/**
1651	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1652	*
1653	* @returns Strict VBox status code.
1654	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1655	* @param pu64 Where to return the opcode qword.
1656	*/
1657	VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1658	{
1659	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1660	if (rcStrict == VINF_SUCCESS)
1661	{
1662	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1663	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1664	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1665	# else
1666	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1667	pVCpu->iem.s.abOpcode[offOpcode + 1],
1668	pVCpu->iem.s.abOpcode[offOpcode + 2],
1669	pVCpu->iem.s.abOpcode[offOpcode + 3],
1670	pVCpu->iem.s.abOpcode[offOpcode + 4],
1671	pVCpu->iem.s.abOpcode[offOpcode + 5],
1672	pVCpu->iem.s.abOpcode[offOpcode + 6],
1673	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1674	# endif
1675	pVCpu->iem.s.offOpcode = offOpcode + 8;
1676	}
1677	else
1678	*pu64 = 0;
1679	return rcStrict;
1680	}
1681
1682	#else /* IEM_WITH_SETJMP */
1683
1684	/**
1685	* Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
1686	*
1687	* @returns The opcode qword.
1688	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1689	*/
1690	uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1691	{
1692	# ifdef IEM_WITH_CODE_TLB
1693	uint64_t u64;
1694	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
1695	return u64;
1696	# else
1697	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1698	if (rcStrict == VINF_SUCCESS)
1699	{
1700	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1701	pVCpu->iem.s.offOpcode = offOpcode + 8;
1702	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1703	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1704	# else
1705	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1706	pVCpu->iem.s.abOpcode[offOpcode + 1],
1707	pVCpu->iem.s.abOpcode[offOpcode + 2],
1708	pVCpu->iem.s.abOpcode[offOpcode + 3],
1709	pVCpu->iem.s.abOpcode[offOpcode + 4],
1710	pVCpu->iem.s.abOpcode[offOpcode + 5],
1711	pVCpu->iem.s.abOpcode[offOpcode + 6],
1712	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1713	# endif
1714	}
1715	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
1716	# endif
1717	}
1718
1719	#endif /* IEM_WITH_SETJMP */
1720
1721
1722
1723	/** @name Misc Worker Functions.
1724	* @{
1725	*/
1726
1727	/**
1728	* Gets the exception class for the specified exception vector.
1729	*
1730	* @returns The class of the specified exception.
1731	* @param uVector The exception vector.
1732	*/
1733	static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT
1734	{
1735	Assert(uVector <= X86_XCPT_LAST);
1736	switch (uVector)
1737	{
1738	case X86_XCPT_DE:
1739	case X86_XCPT_TS:
1740	case X86_XCPT_NP:
1741	case X86_XCPT_SS:
1742	case X86_XCPT_GP:
1743	case X86_XCPT_SX: /* AMD only */
1744	return IEMXCPTCLASS_CONTRIBUTORY;
1745
1746	case X86_XCPT_PF:
1747	case X86_XCPT_VE: /* Intel only */
1748	return IEMXCPTCLASS_PAGE_FAULT;
1749
1750	case X86_XCPT_DF:
1751	return IEMXCPTCLASS_DOUBLE_FAULT;
1752	}
1753	return IEMXCPTCLASS_BENIGN;
1754	}
1755
1756
1757	/**
1758	* Evaluates how to handle an exception caused during delivery of another event
1759	* (exception / interrupt).
1760	*
1761	* @returns How to handle the recursive exception.
1762	* @param pVCpu The cross context virtual CPU structure of the
1763	* calling thread.
1764	* @param fPrevFlags The flags of the previous event.
1765	* @param uPrevVector The vector of the previous event.
1766	* @param fCurFlags The flags of the current exception.
1767	* @param uCurVector The vector of the current exception.
1768	* @param pfXcptRaiseInfo Where to store additional information about the
1769	* exception condition. Optional.
1770	*/
1771	VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
1772	uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
1773	{
1774	/*
1775	* Only CPU exceptions can be raised while delivering other events, software interrupt
1776	* (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
1777	*/
1778	AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
1779	Assert(pVCpu); RT_NOREF(pVCpu);
1780	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
1781
1782	IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
1783	IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
1784	if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1785	{
1786	IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
1787	if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
1788	{
1789	IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
1790	if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
1791	&& ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
1792	\|\| enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
1793	{
1794	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1795	fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
1796	: IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
1797	Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
1798	uCurVector, pVCpu->cpum.GstCtx.cr2));
1799	}
1800	else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1801	&& enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
1802	{
1803	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1804	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
1805	}
1806	else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
1807	&& ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1808	\|\| enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
1809	{
1810	enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
1811	Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
1812	}
1813	}
1814	else
1815	{
1816	if (uPrevVector == X86_XCPT_NMI)
1817	{
1818	fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
1819	if (uCurVector == X86_XCPT_PF)
1820	{
1821	fRaiseInfo \|= IEMXCPTRAISEINFO_NMI_PF;
1822	Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
1823	}
1824	}
1825	else if ( uPrevVector == X86_XCPT_AC
1826	&& uCurVector == X86_XCPT_AC)
1827	{
1828	enmRaise = IEMXCPTRAISE_CPU_HANG;
1829	fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
1830	Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
1831	}
1832	}
1833	}
1834	else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
1835	{
1836	fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
1837	if (uCurVector == X86_XCPT_PF)
1838	fRaiseInfo \|= IEMXCPTRAISEINFO_EXT_INT_PF;
1839	}
1840	else
1841	{
1842	Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
1843	fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
1844	}
1845
1846	if (pfXcptRaiseInfo)
1847	*pfXcptRaiseInfo = fRaiseInfo;
1848	return enmRaise;
1849	}
1850
1851
1852	/**
1853	* Enters the CPU shutdown state initiated by a triple fault or other
1854	* unrecoverable conditions.
1855	*
1856	* @returns Strict VBox status code.
1857	* @param pVCpu The cross context virtual CPU structure of the
1858	* calling thread.
1859	*/
1860	static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT
1861	{
1862	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
1863	IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
1864
1865	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
1866	{
1867	Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
1868	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
1869	}
1870
1871	RT_NOREF(pVCpu);
1872	return VINF_EM_TRIPLE_FAULT;
1873	}
1874
1875
1876	/**
1877	* Validates a new SS segment.
1878	*
1879	* @returns VBox strict status code.
1880	* @param pVCpu The cross context virtual CPU structure of the
1881	* calling thread.
1882	* @param NewSS The new SS selctor.
1883	* @param uCpl The CPL to load the stack for.
1884	* @param pDesc Where to return the descriptor.
1885	*/
1886	static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT
1887	{
1888	/* Null selectors are not allowed (we're not called for dispatching
1889	interrupts with SS=0 in long mode). */
1890	if (!(NewSS & X86_SEL_MASK_OFF_RPL))
1891	{
1892	Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
1893	return iemRaiseTaskSwitchFault0(pVCpu);
1894	}
1895
1896	/** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
1897	if ((NewSS & X86_SEL_RPL) != uCpl)
1898	{
1899	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
1900	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1901	}
1902
1903	/*
1904	* Read the descriptor.
1905	*/
1906	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
1907	if (rcStrict != VINF_SUCCESS)
1908	return rcStrict;
1909
1910	/*
1911	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1912	*/
1913	if (!pDesc->Legacy.Gen.u1DescType)
1914	{
1915	Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1916	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1917	}
1918
1919	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1920	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1921	{
1922	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1923	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1924	}
1925	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1926	{
1927	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1928	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1929	}
1930
1931	/* Is it there? */
1932	/** @todo testcase: Is this checked before the canonical / limit check below? */
1933	if (!pDesc->Legacy.Gen.u1Present)
1934	{
1935	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1936	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
1937	}
1938
1939	return VINF_SUCCESS;
1940	}
1941
1942	/** @} */
1943
1944
1945	/** @name Raising Exceptions.
1946	*
1947	* @{
1948	*/
1949
1950
1951	/**
1952	* Loads the specified stack far pointer from the TSS.
1953	*
1954	* @returns VBox strict status code.
1955	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1956	* @param uCpl The CPL to load the stack for.
1957	* @param pSelSS Where to return the new stack segment.
1958	* @param puEsp Where to return the new stack pointer.
1959	*/
1960	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT
1961	{
1962	VBOXSTRICTRC rcStrict;
1963	Assert(uCpl < 4);
1964
1965	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1966	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
1967	{
1968	/*
1969	* 16-bit TSS (X86TSS16).
1970	*/
1971	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1972	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1973	{
1974	uint32_t off = uCpl * 4 + 2;
1975	if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1976	{
1977	/** @todo check actual access pattern here. */
1978	uint32_t u32Tmp = 0; /* gcc maybe... */
1979	rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1980	if (rcStrict == VINF_SUCCESS)
1981	{
1982	*puEsp = RT_LOWORD(u32Tmp);
1983	*pSelSS = RT_HIWORD(u32Tmp);
1984	return VINF_SUCCESS;
1985	}
1986	}
1987	else
1988	{
1989	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1990	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1991	}
1992	break;
1993	}
1994
1995	/*
1996	* 32-bit TSS (X86TSS32).
1997	*/
1998	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1999	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
2000	{
2001	uint32_t off = uCpl * 8 + 4;
2002	if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
2003	{
2004	/** @todo check actual access pattern here. */
2005	uint64_t u64Tmp;
2006	rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
2007	if (rcStrict == VINF_SUCCESS)
2008	{
2009	*puEsp = u64Tmp & UINT32_MAX;
2010	*pSelSS = (RTSEL)(u64Tmp >> 32);
2011	return VINF_SUCCESS;
2012	}
2013	}
2014	else
2015	{
2016	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
2017	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
2018	}
2019	break;
2020	}
2021
2022	default:
2023	AssertFailed();
2024	rcStrict = VERR_IEM_IPE_4;
2025	break;
2026	}
2027
2028	puEsp = 0; / make gcc happy */
2029	pSelSS = 0; / make gcc happy */
2030	return rcStrict;
2031	}
2032
2033
2034	/**
2035	* Loads the specified stack pointer from the 64-bit TSS.
2036	*
2037	* @returns VBox strict status code.
2038	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2039	* @param uCpl The CPL to load the stack for.
2040	* @param uIst The interrupt stack table index, 0 if to use uCpl.
2041	* @param puRsp Where to return the new stack pointer.
2042	*/
2043	static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT
2044	{
2045	Assert(uCpl < 4);
2046	Assert(uIst < 8);
2047	puRsp = 0; / make gcc happy */
2048
2049	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
2050	AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
2051
2052	uint32_t off;
2053	if (uIst)
2054	off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
2055	else
2056	off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
2057	if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
2058	{
2059	Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
2060	return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
2061	}
2062
2063	return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
2064	}
2065
2066
2067	/**
2068	* Adjust the CPU state according to the exception being raised.
2069	*
2070	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2071	* @param u8Vector The exception that has been raised.
2072	*/
2073	DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
2074	{
2075	switch (u8Vector)
2076	{
2077	case X86_XCPT_DB:
2078	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
2079	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
2080	break;
2081	/** @todo Read the AMD and Intel exception reference... */
2082	}
2083	}
2084
2085
2086	/**
2087	* Implements exceptions and interrupts for real mode.
2088	*
2089	* @returns VBox strict status code.
2090	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2091	* @param cbInstr The number of bytes to offset rIP by in the return
2092	* address.
2093	* @param u8Vector The interrupt / exception vector number.
2094	* @param fFlags The flags.
2095	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2096	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2097	*/
2098	static VBOXSTRICTRC
2099	iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
2100	uint8_t cbInstr,
2101	uint8_t u8Vector,
2102	uint32_t fFlags,
2103	uint16_t uErr,
2104	uint64_t uCr2) RT_NOEXCEPT
2105	{
2106	NOREF(uErr); NOREF(uCr2);
2107	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2108
2109	/*
2110	* Read the IDT entry.
2111	*/
2112	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
2113	{
2114	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
2115	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2116	}
2117	RTFAR16 Idte;
2118	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t )&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) u8Vector);
2119	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2120	{
2121	Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
2122	return rcStrict;
2123	}
2124
2125	#ifdef LOG_ENABLED
2126	/* If software interrupt, try decode it if logging is enabled and such. */
2127	if ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2128	&& LogIsItEnabled(RTLOGGRPFLAGS_ENABLED, LOG_GROUP_IEM_SYSCALL))
2129	iemLogSyscallRealModeInt(pVCpu, u8Vector, cbInstr);
2130	#endif
2131
2132	/*
2133	* Push the stack frame.
2134	*/
2135	uint8_t bUnmapInfo;
2136	uint16_t *pu16Frame;
2137	uint64_t uNewRsp;
2138	rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &bUnmapInfo, &uNewRsp);
2139	if (rcStrict != VINF_SUCCESS)
2140	return rcStrict;
2141
2142	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
2143	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
2144	AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
2145	if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
2146	fEfl \|= UINT16_C(0xf000);
2147	#endif
2148	pu16Frame[2] = (uint16_t)fEfl;
2149	pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
2150	pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
2151	rcStrict = iemMemStackPushCommitSpecial(pVCpu, bUnmapInfo, uNewRsp);
2152	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2153	return rcStrict;
2154
2155	/*
2156	* Load the vector address into cs:ip and make exception specific state
2157	* adjustments.
2158	*/
2159	pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
2160	pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
2161	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2162	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
2163	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
2164	pVCpu->cpum.GstCtx.rip = Idte.off;
2165	fEfl &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_AC);
2166	IEMMISC_SET_EFL(pVCpu, fEfl);
2167
2168	/** @todo do we actually do this in real mode? */
2169	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
2170	iemRaiseXcptAdjustState(pVCpu, u8Vector);
2171
2172	/* The IEM_F_MODE_XXX and IEM_F_X86_CPL_MASK doesn't really change here,
2173	so best leave them alone in case we're in a weird kind of real mode... */
2174
2175	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2176	}
2177
2178
2179	/**
2180	* Loads a NULL data selector into when coming from V8086 mode.
2181	*
2182	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2183	* @param pSReg Pointer to the segment register.
2184	*/
2185	DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
2186	{
2187	pSReg->Sel = 0;
2188	pSReg->ValidSel = 0;
2189	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2190	{
2191	/* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
2192	pSReg->Attr.u &= X86DESCATTR_DT \| X86DESCATTR_TYPE \| X86DESCATTR_DPL \| X86DESCATTR_G \| X86DESCATTR_D;
2193	pSReg->Attr.u \|= X86DESCATTR_UNUSABLE;
2194	}
2195	else
2196	{
2197	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2198	/** @todo check this on AMD-V */
2199	pSReg->u64Base = 0;
2200	pSReg->u32Limit = 0;
2201	}
2202	}
2203
2204
2205	/**
2206	* Loads a segment selector during a task switch in V8086 mode.
2207	*
2208	* @param pSReg Pointer to the segment register.
2209	* @param uSel The selector value to load.
2210	*/
2211	DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
2212	{
2213	/* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
2214	pSReg->Sel = uSel;
2215	pSReg->ValidSel = uSel;
2216	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2217	pSReg->u64Base = uSel << 4;
2218	pSReg->u32Limit = 0xffff;
2219	pSReg->Attr.u = 0xf3;
2220	}
2221
2222
2223	/**
2224	* Loads a segment selector during a task switch in protected mode.
2225	*
2226	* In this task switch scenario, we would throw \#TS exceptions rather than
2227	* \#GPs.
2228	*
2229	* @returns VBox strict status code.
2230	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2231	* @param pSReg Pointer to the segment register.
2232	* @param uSel The new selector value.
2233	*
2234	* @remarks This does _not_ handle CS or SS.
2235	* @remarks This expects IEM_GET_CPL(pVCpu) to return an up to date value.
2236	*/
2237	static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT
2238	{
2239	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2240
2241	/* Null data selector. */
2242	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2243	{
2244	iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
2245	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2246	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2247	return VINF_SUCCESS;
2248	}
2249
2250	/* Fetch the descriptor. */
2251	IEMSELDESC Desc;
2252	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
2253	if (rcStrict != VINF_SUCCESS)
2254	{
2255	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
2256	VBOXSTRICTRC_VAL(rcStrict)));
2257	return rcStrict;
2258	}
2259
2260	/* Must be a data segment or readable code segment. */
2261	if ( !Desc.Legacy.Gen.u1DescType
2262	\|\| (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2263	{
2264	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
2265	Desc.Legacy.Gen.u4Type));
2266	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2267	}
2268
2269	/* Check privileges for data segments and non-conforming code segments. */
2270	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2271	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2272	{
2273	/* The RPL and the new CPL must be less than or equal to the DPL. */
2274	if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
2275	\|\| (IEM_GET_CPL(pVCpu) > Desc.Legacy.Gen.u2Dpl))
2276	{
2277	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
2278	uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
2279	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2280	}
2281	}
2282
2283	/* Is it there? */
2284	if (!Desc.Legacy.Gen.u1Present)
2285	{
2286	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
2287	return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2288	}
2289
2290	/* The base and limit. */
2291	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2292	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
2293
2294	/*
2295	* Ok, everything checked out fine. Now set the accessed bit before
2296	* committing the result into the registers.
2297	*/
2298	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2299	{
2300	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2301	if (rcStrict != VINF_SUCCESS)
2302	return rcStrict;
2303	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2304	}
2305
2306	/* Commit */
2307	pSReg->Sel = uSel;
2308	pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2309	pSReg->u32Limit = cbLimit;
2310	pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
2311	pSReg->ValidSel = uSel;
2312	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2313	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2314	pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
2315
2316	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2317	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2318	return VINF_SUCCESS;
2319	}
2320
2321
2322	/**
2323	* Performs a task switch.
2324	*
2325	* If the task switch is the result of a JMP, CALL or IRET instruction, the
2326	* caller is responsible for performing the necessary checks (like DPL, TSS
2327	* present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
2328	* reference for JMP, CALL, IRET.
2329	*
2330	* If the task switch is the due to a software interrupt or hardware exception,
2331	* the caller is responsible for validating the TSS selector and descriptor. See
2332	* Intel Instruction reference for INT n.
2333	*
2334	* @returns VBox strict status code.
2335	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2336	* @param enmTaskSwitch The cause of the task switch.
2337	* @param uNextEip The EIP effective after the task switch.
2338	* @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
2339	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2340	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2341	* @param SelTss The TSS selector of the new task.
2342	* @param pNewDescTss Pointer to the new TSS descriptor.
2343	*/
2344	VBOXSTRICTRC
2345	iemTaskSwitch(PVMCPUCC pVCpu,
2346	IEMTASKSWITCH enmTaskSwitch,
2347	uint32_t uNextEip,
2348	uint32_t fFlags,
2349	uint16_t uErr,
2350	uint64_t uCr2,
2351	RTSEL SelTss,
2352	PIEMSELDESC pNewDescTss) RT_NOEXCEPT
2353	{
2354	Assert(!IEM_IS_REAL_MODE(pVCpu));
2355	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2356	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2357
2358	uint32_t const uNewTssType = pNewDescTss->Legacy.Gate.u4Type;
2359	Assert( uNewTssType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2360	\|\| uNewTssType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2361	\|\| uNewTssType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2362	\|\| uNewTssType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2363
2364	bool const fIsNewTss386 = ( uNewTssType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2365	\|\| uNewTssType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2366
2367	Log(("iemTaskSwitch: enmTaskSwitch=%u NewTss=%#x fIsNewTss386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTss,
2368	fIsNewTss386, pVCpu->cpum.GstCtx.eip, uNextEip));
2369
2370	/* Update CR2 in case it's a page-fault. */
2371	/** @todo This should probably be done much earlier in IEM/PGM. See
2372	* @bugref{5653#c49}. */
2373	if (fFlags & IEM_XCPT_FLAGS_CR2)
2374	pVCpu->cpum.GstCtx.cr2 = uCr2;
2375
2376	/*
2377	* Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
2378	* subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
2379	*/
2380	uint32_t const uNewTssLimit = pNewDescTss->Legacy.Gen.u16LimitLow \| (pNewDescTss->Legacy.Gen.u4LimitHigh << 16);
2381	uint32_t const uNewTssLimitMin = fIsNewTss386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
2382	if (uNewTssLimit < uNewTssLimitMin)
2383	{
2384	Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTssLimit=%#x uNewTssLimitMin=%#x -> #TS\n",
2385	enmTaskSwitch, uNewTssLimit, uNewTssLimitMin));
2386	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTss & X86_SEL_MASK_OFF_RPL);
2387	}
2388
2389	/*
2390	* Task switches in VMX non-root mode always cause task switches.
2391	* The new TSS must have been read and validated (DPL, limits etc.) before a
2392	* task-switch VM-exit commences.
2393	*
2394	* See Intel spec. 25.4.2 "Treatment of Task Switches".
2395	*/
2396	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
2397	{
2398	Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTss));
2399	IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTss, uNextEip - pVCpu->cpum.GstCtx.eip);
2400	}
2401
2402	/*
2403	* The SVM nested-guest intercept for task-switch takes priority over all exceptions
2404	* after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
2405	*/
2406	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
2407	{
2408	uint64_t const uExitInfo1 = SelTss;
2409	uint64_t uExitInfo2 = uErr;
2410	switch (enmTaskSwitch)
2411	{
2412	case IEMTASKSWITCH_JUMP: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_JUMP; break;
2413	case IEMTASKSWITCH_IRET: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_IRET; break;
2414	default: break;
2415	}
2416	if (fFlags & IEM_XCPT_FLAGS_ERR)
2417	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
2418	if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
2419	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
2420
2421	Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
2422	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
2423	RT_NOREF2(uExitInfo1, uExitInfo2);
2424	}
2425
2426	/*
2427	* Check the current TSS limit. The last written byte to the current TSS during the
2428	* task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
2429	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2430	*
2431	* The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
2432	* end up with smaller than "legal" TSS limits.
2433	*/
2434	uint32_t const uCurTssLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
2435	uint32_t const uCurTssLimitMin = fIsNewTss386 ? 0x5F : 0x29;
2436	if (uCurTssLimit < uCurTssLimitMin)
2437	{
2438	Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTssLimit=%#x uCurTssLimitMin=%#x -> #TS\n",
2439	enmTaskSwitch, uCurTssLimit, uCurTssLimitMin));
2440	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTss & X86_SEL_MASK_OFF_RPL);
2441	}
2442
2443	/*
2444	* Verify that the new TSS can be accessed and map it. Map only the required contents
2445	* and not the entire TSS.
2446	*/
2447	uint8_t bUnmapInfoNewTss;
2448	void *pvNewTss;
2449	uint32_t const cbNewTss = uNewTssLimitMin + 1;
2450	RTGCPTR const GCPtrNewTss = X86DESC_BASE(&pNewDescTss->Legacy);
2451	AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
2452	/** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
2453	* not perform correct translation if this happens. See Intel spec. 7.2.1
2454	* "Task-State Segment". */
2455	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTss, &bUnmapInfoNewTss, cbNewTss, UINT8_MAX, GCPtrNewTss, IEM_ACCESS_SYS_RW, 0);
2456	/** @todo Not cleaning up bUnmapInfoNewTss mapping in any early exits here.
2457	* Consider wrapping the remainder into a function for simpler cleanup. */
2458	if (rcStrict != VINF_SUCCESS)
2459	{
2460	Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTss=%u uNewTssLimit=%u rc=%Rrc\n", enmTaskSwitch,
2461	cbNewTss, uNewTssLimit, VBOXSTRICTRC_VAL(rcStrict)));
2462	return rcStrict;
2463	}
2464
2465	/*
2466	* Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
2467	*/
2468	uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
2469	if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
2470	\|\| enmTaskSwitch == IEMTASKSWITCH_IRET)
2471	{
2472	uint8_t bUnmapInfoDescCurTss;
2473	PX86DESC pDescCurTss;
2474	rcStrict = iemMemMap(pVCpu, (void *)&pDescCurTss, &bUnmapInfoDescCurTss, sizeof(pDescCurTss), UINT8_MAX,
2475	pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2476	if (rcStrict != VINF_SUCCESS)
2477	{
2478	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2479	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2480	return rcStrict;
2481	}
2482
2483	pDescCurTss->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2484	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoDescCurTss);
2485	if (rcStrict != VINF_SUCCESS)
2486	{
2487	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2488	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2489	return rcStrict;
2490	}
2491
2492	/* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
2493	if (enmTaskSwitch == IEMTASKSWITCH_IRET)
2494	{
2495	Assert( uNewTssType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2496	\|\| uNewTssType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2497	fEFlags &= ~X86_EFL_NT;
2498	}
2499	}
2500
2501	/*
2502	* Save the CPU state into the current TSS.
2503	*/
2504	RTGCPTR const GCPtrCurTss = pVCpu->cpum.GstCtx.tr.u64Base;
2505	if (GCPtrNewTss == GCPtrCurTss)
2506	{
2507	Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur\|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTss));
2508	Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
2509	pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, pVCpu->cpum.GstCtx.eax,
2510	pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
2511	pVCpu->cpum.GstCtx.ldtr.Sel));
2512	}
2513	if (fIsNewTss386)
2514	{
2515	/*
2516	* Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
2517	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2518	*/
2519	uint8_t bUnmapInfoCurTss32;
2520	void *pvCurTss32;
2521	uint32_t const offCurTss = RT_UOFFSETOF(X86TSS32, eip);
2522	uint32_t const cbCurTss = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
2523	AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
2524	rcStrict = iemMemMap(pVCpu, &pvCurTss32, &bUnmapInfoCurTss32, cbCurTss, UINT8_MAX,
2525	GCPtrCurTss + offCurTss, IEM_ACCESS_SYS_RW, 0);
2526	if (rcStrict != VINF_SUCCESS)
2527	{
2528	Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTss=%#RGv cb=%u rc=%Rrc\n",
2529	enmTaskSwitch, GCPtrCurTss, cbCurTss, VBOXSTRICTRC_VAL(rcStrict)));
2530	return rcStrict;
2531	}
2532
2533	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTss..cbCurTss). */
2534	PX86TSS32 pCurTss32 = (PX86TSS32)((uintptr_t)pvCurTss32 - offCurTss);
2535	pCurTss32->eip = uNextEip;
2536	pCurTss32->eflags = fEFlags;
2537	pCurTss32->eax = pVCpu->cpum.GstCtx.eax;
2538	pCurTss32->ecx = pVCpu->cpum.GstCtx.ecx;
2539	pCurTss32->edx = pVCpu->cpum.GstCtx.edx;
2540	pCurTss32->ebx = pVCpu->cpum.GstCtx.ebx;
2541	pCurTss32->esp = pVCpu->cpum.GstCtx.esp;
2542	pCurTss32->ebp = pVCpu->cpum.GstCtx.ebp;
2543	pCurTss32->esi = pVCpu->cpum.GstCtx.esi;
2544	pCurTss32->edi = pVCpu->cpum.GstCtx.edi;
2545	pCurTss32->es = pVCpu->cpum.GstCtx.es.Sel;
2546	pCurTss32->cs = pVCpu->cpum.GstCtx.cs.Sel;
2547	pCurTss32->ss = pVCpu->cpum.GstCtx.ss.Sel;
2548	pCurTss32->ds = pVCpu->cpum.GstCtx.ds.Sel;
2549	pCurTss32->fs = pVCpu->cpum.GstCtx.fs.Sel;
2550	pCurTss32->gs = pVCpu->cpum.GstCtx.gs.Sel;
2551
2552	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoCurTss32);
2553	if (rcStrict != VINF_SUCCESS)
2554	{
2555	Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2556	VBOXSTRICTRC_VAL(rcStrict)));
2557	return rcStrict;
2558	}
2559	}
2560	else
2561	{
2562	/*
2563	* Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
2564	*/
2565	uint8_t bUnmapInfoCurTss16;
2566	void *pvCurTss16;
2567	uint32_t const offCurTss = RT_UOFFSETOF(X86TSS16, ip);
2568	uint32_t const cbCurTss = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
2569	AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
2570	rcStrict = iemMemMap(pVCpu, &pvCurTss16, &bUnmapInfoCurTss16, cbCurTss, UINT8_MAX,
2571	GCPtrCurTss + offCurTss, IEM_ACCESS_SYS_RW, 0);
2572	if (rcStrict != VINF_SUCCESS)
2573	{
2574	Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTss=%#RGv cb=%u rc=%Rrc\n",
2575	enmTaskSwitch, GCPtrCurTss, cbCurTss, VBOXSTRICTRC_VAL(rcStrict)));
2576	return rcStrict;
2577	}
2578
2579	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTss..cbCurTss). */
2580	PX86TSS16 pCurTss16 = (PX86TSS16)((uintptr_t)pvCurTss16 - offCurTss);
2581	pCurTss16->ip = uNextEip;
2582	pCurTss16->flags = (uint16_t)fEFlags;
2583	pCurTss16->ax = pVCpu->cpum.GstCtx.ax;
2584	pCurTss16->cx = pVCpu->cpum.GstCtx.cx;
2585	pCurTss16->dx = pVCpu->cpum.GstCtx.dx;
2586	pCurTss16->bx = pVCpu->cpum.GstCtx.bx;
2587	pCurTss16->sp = pVCpu->cpum.GstCtx.sp;
2588	pCurTss16->bp = pVCpu->cpum.GstCtx.bp;
2589	pCurTss16->si = pVCpu->cpum.GstCtx.si;
2590	pCurTss16->di = pVCpu->cpum.GstCtx.di;
2591	pCurTss16->es = pVCpu->cpum.GstCtx.es.Sel;
2592	pCurTss16->cs = pVCpu->cpum.GstCtx.cs.Sel;
2593	pCurTss16->ss = pVCpu->cpum.GstCtx.ss.Sel;
2594	pCurTss16->ds = pVCpu->cpum.GstCtx.ds.Sel;
2595
2596	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoCurTss16);
2597	if (rcStrict != VINF_SUCCESS)
2598	{
2599	Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2600	VBOXSTRICTRC_VAL(rcStrict)));
2601	return rcStrict;
2602	}
2603	}
2604
2605	/*
2606	* Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
2607	*/
2608	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2609	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2610	{
2611	/* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
2612	PX86TSS32 pNewTSS = (PX86TSS32)pvNewTss;
2613	pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
2614	}
2615
2616	/*
2617	* Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
2618	* it's done further below with error handling (e.g. CR3 changes will go through PGM).
2619	*/
2620	uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
2621	uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
2622	bool fNewDebugTrap;
2623	if (fIsNewTss386)
2624	{
2625	PCX86TSS32 pNewTss32 = (PCX86TSS32)pvNewTss;
2626	uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTss32->cr3 : 0;
2627	uNewEip = pNewTss32->eip;
2628	uNewEflags = pNewTss32->eflags;
2629	uNewEax = pNewTss32->eax;
2630	uNewEcx = pNewTss32->ecx;
2631	uNewEdx = pNewTss32->edx;
2632	uNewEbx = pNewTss32->ebx;
2633	uNewEsp = pNewTss32->esp;
2634	uNewEbp = pNewTss32->ebp;
2635	uNewEsi = pNewTss32->esi;
2636	uNewEdi = pNewTss32->edi;
2637	uNewES = pNewTss32->es;
2638	uNewCS = pNewTss32->cs;
2639	uNewSS = pNewTss32->ss;
2640	uNewDS = pNewTss32->ds;
2641	uNewFS = pNewTss32->fs;
2642	uNewGS = pNewTss32->gs;
2643	uNewLdt = pNewTss32->selLdt;
2644	fNewDebugTrap = RT_BOOL(pNewTss32->fDebugTrap);
2645	}
2646	else
2647	{
2648	PCX86TSS16 pNewTss16 = (PCX86TSS16)pvNewTss;
2649	uNewCr3 = 0;
2650	uNewEip = pNewTss16->ip;
2651	uNewEflags = pNewTss16->flags;
2652	uNewEax = UINT32_C(0xffff0000) \| pNewTss16->ax;
2653	uNewEcx = UINT32_C(0xffff0000) \| pNewTss16->cx;
2654	uNewEdx = UINT32_C(0xffff0000) \| pNewTss16->dx;
2655	uNewEbx = UINT32_C(0xffff0000) \| pNewTss16->bx;
2656	uNewEsp = UINT32_C(0xffff0000) \| pNewTss16->sp;
2657	uNewEbp = UINT32_C(0xffff0000) \| pNewTss16->bp;
2658	uNewEsi = UINT32_C(0xffff0000) \| pNewTss16->si;
2659	uNewEdi = UINT32_C(0xffff0000) \| pNewTss16->di;
2660	uNewES = pNewTss16->es;
2661	uNewCS = pNewTss16->cs;
2662	uNewSS = pNewTss16->ss;
2663	uNewDS = pNewTss16->ds;
2664	uNewFS = 0;
2665	uNewGS = 0;
2666	uNewLdt = pNewTss16->selLdt;
2667	fNewDebugTrap = false;
2668	}
2669
2670	if (GCPtrNewTss == GCPtrCurTss)
2671	Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
2672	uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
2673
2674	/*
2675	* We're done accessing the new TSS.
2676	*/
2677	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoNewTss);
2678	if (rcStrict != VINF_SUCCESS)
2679	{
2680	Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
2681	return rcStrict;
2682	}
2683
2684	/*
2685	* Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
2686	*/
2687	if (enmTaskSwitch != IEMTASKSWITCH_IRET)
2688	{
2689	rcStrict = iemMemMap(pVCpu, (void *)&pNewDescTss, &bUnmapInfoNewTss, sizeof(pNewDescTss), UINT8_MAX,
2690	pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTss & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2691	if (rcStrict != VINF_SUCCESS)
2692	{
2693	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2694	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2695	return rcStrict;
2696	}
2697
2698	/* Check that the descriptor indicates the new TSS is available (not busy). */
2699	AssertMsg( pNewDescTss->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2700	\|\| pNewDescTss->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
2701	("Invalid TSS descriptor type=%#x", pNewDescTss->Legacy.Gate.u4Type));
2702
2703	pNewDescTss->Legacy.Gate.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2704	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoNewTss);
2705	if (rcStrict != VINF_SUCCESS)
2706	{
2707	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2708	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2709	return rcStrict;
2710	}
2711	}
2712
2713	/*
2714	* From this point on, we're technically in the new task. We will defer exceptions
2715	* until the completion of the task switch but before executing any instructions in the new task.
2716	*/
2717	pVCpu->cpum.GstCtx.tr.Sel = SelTss;
2718	pVCpu->cpum.GstCtx.tr.ValidSel = SelTss;
2719	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
2720	pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTss->Legacy);
2721	pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTss->Legacy);
2722	pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTss->Legacy);
2723	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
2724
2725	/* Set the busy bit in TR. */
2726	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2727
2728	/* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
2729	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2730	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2731	{
2732	uNewEflags \|= X86_EFL_NT;
2733	}
2734
2735	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
2736	pVCpu->cpum.GstCtx.cr0 \|= X86_CR0_TS;
2737	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
2738
2739	pVCpu->cpum.GstCtx.eip = uNewEip;
2740	pVCpu->cpum.GstCtx.eax = uNewEax;
2741	pVCpu->cpum.GstCtx.ecx = uNewEcx;
2742	pVCpu->cpum.GstCtx.edx = uNewEdx;
2743	pVCpu->cpum.GstCtx.ebx = uNewEbx;
2744	pVCpu->cpum.GstCtx.esp = uNewEsp;
2745	pVCpu->cpum.GstCtx.ebp = uNewEbp;
2746	pVCpu->cpum.GstCtx.esi = uNewEsi;
2747	pVCpu->cpum.GstCtx.edi = uNewEdi;
2748
2749	uNewEflags &= X86_EFL_LIVE_MASK;
2750	uNewEflags \|= X86_EFL_RA1_MASK;
2751	IEMMISC_SET_EFL(pVCpu, uNewEflags);
2752
2753	/*
2754	* Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
2755	* will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
2756	* due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
2757	*/
2758	pVCpu->cpum.GstCtx.es.Sel = uNewES;
2759	pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
2760
2761	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2762	pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
2763
2764	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2765	pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
2766
2767	pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
2768	pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
2769
2770	pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
2771	pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
2772
2773	pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
2774	pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
2775	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2776
2777	pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
2778	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
2779	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
2780	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
2781
2782	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2783	{
2784	pVCpu->cpum.GstCtx.es.Attr.u \|= X86DESCATTR_UNUSABLE;
2785	pVCpu->cpum.GstCtx.cs.Attr.u \|= X86DESCATTR_UNUSABLE;
2786	pVCpu->cpum.GstCtx.ss.Attr.u \|= X86DESCATTR_UNUSABLE;
2787	pVCpu->cpum.GstCtx.ds.Attr.u \|= X86DESCATTR_UNUSABLE;
2788	pVCpu->cpum.GstCtx.fs.Attr.u \|= X86DESCATTR_UNUSABLE;
2789	pVCpu->cpum.GstCtx.gs.Attr.u \|= X86DESCATTR_UNUSABLE;
2790	pVCpu->cpum.GstCtx.ldtr.Attr.u \|= X86DESCATTR_UNUSABLE;
2791	}
2792
2793	/*
2794	* Switch CR3 for the new task.
2795	*/
2796	if ( fIsNewTss386
2797	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
2798	{
2799	/** @todo Should we update and flush TLBs only if CR3 value actually changes? */
2800	int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
2801	AssertRCSuccessReturn(rc, rc);
2802
2803	/* Inform PGM. */
2804	/** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
2805	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
2806	AssertRCReturn(rc, rc);
2807	/* ignore informational status codes */
2808
2809	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
2810	}
2811
2812	/*
2813	* Switch LDTR for the new task.
2814	*/
2815	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
2816	iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
2817	else
2818	{
2819	Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
2820
2821	IEMSELDESC DescNewLdt;
2822	rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
2823	if (rcStrict != VINF_SUCCESS)
2824	{
2825	Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
2826	uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
2827	return rcStrict;
2828	}
2829	if ( !DescNewLdt.Legacy.Gen.u1Present
2830	\|\| DescNewLdt.Legacy.Gen.u1DescType
2831	\|\| DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
2832	{
2833	Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
2834	uNewLdt, DescNewLdt.Legacy.u));
2835	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
2836	}
2837
2838	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
2839	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
2840	pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
2841	pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
2842	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
2843	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2844	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
2845	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
2846	}
2847
2848	IEMSELDESC DescSS;
2849	if (IEM_IS_V86_MODE(pVCpu))
2850	{
2851	IEM_SET_CPL(pVCpu, 3);
2852	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
2853	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
2854	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
2855	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
2856	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
2857	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
2858
2859	/* Quick fix: fake DescSS. / /* @todo fix the code further down? */
2860	DescSS.Legacy.u = 0;
2861	DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
2862	DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
2863	DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
2864	DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
2865	DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
2866	DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2867	DescSS.Legacy.Gen.u2Dpl = 3;
2868	}
2869	else
2870	{
2871	uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
2872
2873	/*
2874	* Load the stack segment for the new task.
2875	*/
2876	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
2877	{
2878	Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
2879	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2880	}
2881
2882	/* Fetch the descriptor. */
2883	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
2884	if (rcStrict != VINF_SUCCESS)
2885	{
2886	Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
2887	VBOXSTRICTRC_VAL(rcStrict)));
2888	return rcStrict;
2889	}
2890
2891	/* SS must be a data segment and writable. */
2892	if ( !DescSS.Legacy.Gen.u1DescType
2893	\|\| (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2894	\|\| !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
2895	{
2896	Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
2897	uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
2898	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2899	}
2900
2901	/* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
2902	if ( (uNewSS & X86_SEL_RPL) != uNewCpl
2903	\|\| DescSS.Legacy.Gen.u2Dpl != uNewCpl)
2904	{
2905	Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
2906	uNewCpl));
2907	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2908	}
2909
2910	/* Is it there? */
2911	if (!DescSS.Legacy.Gen.u1Present)
2912	{
2913	Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
2914	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2915	}
2916
2917	uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
2918	uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
2919
2920	/* Set the accessed bit before committing the result into SS. */
2921	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2922	{
2923	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
2924	if (rcStrict != VINF_SUCCESS)
2925	return rcStrict;
2926	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2927	}
2928
2929	/* Commit SS. */
2930	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2931	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
2932	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
2933	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
2934	pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
2935	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2936	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
2937
2938	/* CPL has changed, update IEM before loading rest of segments. */
2939	IEM_SET_CPL(pVCpu, uNewCpl);
2940
2941	/*
2942	* Load the data segments for the new task.
2943	*/
2944	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
2945	if (rcStrict != VINF_SUCCESS)
2946	return rcStrict;
2947	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
2948	if (rcStrict != VINF_SUCCESS)
2949	return rcStrict;
2950	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
2951	if (rcStrict != VINF_SUCCESS)
2952	return rcStrict;
2953	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
2954	if (rcStrict != VINF_SUCCESS)
2955	return rcStrict;
2956
2957	/*
2958	* Load the code segment for the new task.
2959	*/
2960	if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
2961	{
2962	Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
2963	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2964	}
2965
2966	/* Fetch the descriptor. */
2967	IEMSELDESC DescCS;
2968	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
2969	if (rcStrict != VINF_SUCCESS)
2970	{
2971	Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
2972	return rcStrict;
2973	}
2974
2975	/* CS must be a code segment. */
2976	if ( !DescCS.Legacy.Gen.u1DescType
2977	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2978	{
2979	Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
2980	DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
2981	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2982	}
2983
2984	/* For conforming CS, DPL must be less than or equal to the RPL. */
2985	if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2986	&& DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
2987	{
2988	Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
2989	DescCS.Legacy.Gen.u2Dpl));
2990	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2991	}
2992
2993	/* For non-conforming CS, DPL must match RPL. */
2994	if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2995	&& DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
2996	{
2997	Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
2998	DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
2999	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3000	}
3001
3002	/* Is it there? */
3003	if (!DescCS.Legacy.Gen.u1Present)
3004	{
3005	Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
3006	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
3007	}
3008
3009	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
3010	u64Base = X86DESC_BASE(&DescCS.Legacy);
3011
3012	/* Set the accessed bit before committing the result into CS. */
3013	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3014	{
3015	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
3016	if (rcStrict != VINF_SUCCESS)
3017	return rcStrict;
3018	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3019	}
3020
3021	/* Commit CS. */
3022	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
3023	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
3024	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3025	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
3026	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
3027	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3028	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
3029	}
3030
3031	/* Make sure the CPU mode is correct. */
3032	uint32_t const fExecNew = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
3033	if (fExecNew != pVCpu->iem.s.fExec)
3034	Log(("iemTaskSwitch: fExec %#x -> %#x (xor %#x)\n", pVCpu->iem.s.fExec, fExecNew, pVCpu->iem.s.fExec ^ fExecNew));
3035	pVCpu->iem.s.fExec = fExecNew;
3036
3037	/** @todo Debug trap. */
3038	if (fIsNewTss386 && fNewDebugTrap)
3039	Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
3040
3041	/*
3042	* Construct the error code masks based on what caused this task switch.
3043	* See Intel Instruction reference for INT.
3044	*/
3045	uint16_t uExt;
3046	if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
3047	&& ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
3048	\|\| (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
3049	uExt = 1;
3050	else
3051	uExt = 0;
3052
3053	/*
3054	* Push any error code on to the new stack.
3055	*/
3056	if (fFlags & IEM_XCPT_FLAGS_ERR)
3057	{
3058	Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
3059	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3060	uint8_t const cbStackFrame = fIsNewTss386 ? 4 : 2;
3061
3062	/* Check that there is sufficient space on the stack. */
3063	/** @todo Factor out segment limit checking for normal/expand down segments
3064	* into a separate function. */
3065	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
3066	{
3067	if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
3068	\|\| pVCpu->cpum.GstCtx.esp < cbStackFrame)
3069	{
3070	/** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
3071	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
3072	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
3073	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
3074	}
3075	}
3076	else
3077	{
3078	if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
3079	\|\| pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
3080	{
3081	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
3082	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
3083	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
3084	}
3085	}
3086
3087
3088	if (fIsNewTss386)
3089	rcStrict = iemMemStackPushU32(pVCpu, uErr);
3090	else
3091	rcStrict = iemMemStackPushU16(pVCpu, uErr);
3092	if (rcStrict != VINF_SUCCESS)
3093	{
3094	Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
3095	fIsNewTss386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
3096	return rcStrict;
3097	}
3098	}
3099
3100	/* Check the new EIP against the new CS limit. */
3101	if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
3102	{
3103	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
3104	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
3105	/** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
3106	return iemRaiseGeneralProtectionFault(pVCpu, uExt);
3107	}
3108
3109	Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
3110	pVCpu->cpum.GstCtx.ss.Sel));
3111	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3112	}
3113
3114
3115	/**
3116	* Implements exceptions and interrupts for protected mode.
3117	*
3118	* @returns VBox strict status code.
3119	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3120	* @param cbInstr The number of bytes to offset rIP by in the return
3121	* address.
3122	* @param u8Vector The interrupt / exception vector number.
3123	* @param fFlags The flags.
3124	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3125	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3126	*/
3127	static VBOXSTRICTRC
3128	iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
3129	uint8_t cbInstr,
3130	uint8_t u8Vector,
3131	uint32_t fFlags,
3132	uint16_t uErr,
3133	uint64_t uCr2) RT_NOEXCEPT
3134	{
3135	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3136
3137	/*
3138	* Read the IDT entry.
3139	*/
3140	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
3141	{
3142	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3143	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3144	}
3145	X86DESC Idte;
3146	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
3147	pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
3148	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3149	{
3150	Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3151	return rcStrict;
3152	}
3153	Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x - from %04x:%08RX64 efl=%#x depth=%d\n",
3154	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3155	Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow,
3156	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags.u, pVCpu->iem.s.cXcptRecursions));
3157
3158	/*
3159	* Check the descriptor type, DPL and such.
3160	* ASSUMES this is done in the same order as described for call-gate calls.
3161	*/
3162	if (Idte.Gate.u1DescType)
3163	{
3164	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3165	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3166	}
3167	bool fTaskGate = false;
3168	uint8_t f32BitGate = true;
3169	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3170	switch (Idte.Gate.u4Type)
3171	{
3172	case X86_SEL_TYPE_SYS_UNDEFINED:
3173	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
3174	case X86_SEL_TYPE_SYS_LDT:
3175	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
3176	case X86_SEL_TYPE_SYS_286_CALL_GATE:
3177	case X86_SEL_TYPE_SYS_UNDEFINED2:
3178	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
3179	case X86_SEL_TYPE_SYS_UNDEFINED3:
3180	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
3181	case X86_SEL_TYPE_SYS_386_CALL_GATE:
3182	case X86_SEL_TYPE_SYS_UNDEFINED4:
3183	{
3184	/** @todo check what actually happens when the type is wrong...
3185	* esp. call gates. */
3186	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3187	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3188	}
3189
3190	case X86_SEL_TYPE_SYS_286_INT_GATE:
3191	f32BitGate = false;
3192	RT_FALL_THRU();
3193	case X86_SEL_TYPE_SYS_386_INT_GATE:
3194	fEflToClear \|= X86_EFL_IF;
3195	break;
3196
3197	case X86_SEL_TYPE_SYS_TASK_GATE:
3198	fTaskGate = true;
3199	#ifndef IEM_IMPLEMENTS_TASKSWITCH
3200	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
3201	#endif
3202	break;
3203
3204	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
3205	f32BitGate = false;
3206	break;
3207	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
3208	break;
3209
3210	IEM_NOT_REACHED_DEFAULT_CASE_RET();
3211	}
3212
3213	/* Check DPL against CPL if applicable. */
3214	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
3215	{
3216	if (IEM_GET_CPL(pVCpu) > Idte.Gate.u2Dpl)
3217	{
3218	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, IEM_GET_CPL(pVCpu), Idte.Gate.u2Dpl));
3219	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3220	}
3221	}
3222
3223	/* Is it there? */
3224	if (!Idte.Gate.u1Present)
3225	{
3226	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
3227	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3228	}
3229
3230	/* Is it a task-gate? */
3231	if (fTaskGate)
3232	{
3233	/*
3234	* Construct the error code masks based on what caused this task switch.
3235	* See Intel Instruction reference for INT.
3236	*/
3237	uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
3238	&& !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
3239	uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
3240	RTSEL SelTss = Idte.Gate.u16Sel;
3241
3242	/*
3243	* Fetch the TSS descriptor in the GDT.
3244	*/
3245	IEMSELDESC DescTSS;
3246	rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTss, X86_XCPT_GP, (SelTss & uSelMask) \| uExt);
3247	if (rcStrict != VINF_SUCCESS)
3248	{
3249	Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTss,
3250	VBOXSTRICTRC_VAL(rcStrict)));
3251	return rcStrict;
3252	}
3253
3254	/* The TSS descriptor must be a system segment and be available (not busy). */
3255	if ( DescTSS.Legacy.Gen.u1DescType
3256	\|\| ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
3257	&& DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
3258	{
3259	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
3260	u8Vector, SelTss, DescTSS.Legacy.au64));
3261	return iemRaiseGeneralProtectionFault(pVCpu, (SelTss & uSelMask) \| uExt);
3262	}
3263
3264	/* The TSS must be present. */
3265	if (!DescTSS.Legacy.Gen.u1Present)
3266	{
3267	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTss, DescTSS.Legacy.au64));
3268	return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTss & uSelMask) \| uExt);
3269	}
3270
3271	/* Do the actual task switch. */
3272	return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
3273	(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
3274	fFlags, uErr, uCr2, SelTss, &DescTSS);
3275	}
3276
3277	/* A null CS is bad. */
3278	RTSEL NewCS = Idte.Gate.u16Sel;
3279	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3280	{
3281	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3282	return iemRaiseGeneralProtectionFault0(pVCpu);
3283	}
3284
3285	/* Fetch the descriptor for the new CS. */
3286	IEMSELDESC DescCS;
3287	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
3288	if (rcStrict != VINF_SUCCESS)
3289	{
3290	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3291	return rcStrict;
3292	}
3293
3294	/* Must be a code segment. */
3295	if (!DescCS.Legacy.Gen.u1DescType)
3296	{
3297	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3298	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3299	}
3300	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3301	{
3302	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3303	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3304	}
3305
3306	/* Don't allow lowering the privilege level. */
3307	/** @todo Does the lowering of privileges apply to software interrupts
3308	* only? This has bearings on the more-privileged or
3309	* same-privilege stack behavior further down. A testcase would
3310	* be nice. */
3311	if (DescCS.Legacy.Gen.u2Dpl > IEM_GET_CPL(pVCpu))
3312	{
3313	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3314	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
3315	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3316	}
3317
3318	/* Make sure the selector is present. */
3319	if (!DescCS.Legacy.Gen.u1Present)
3320	{
3321	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3322	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3323	}
3324
3325	#ifdef LOG_ENABLED
3326	/* If software interrupt, try decode it if logging is enabled and such. */
3327	if ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
3328	&& LogIsItEnabled(RTLOGGRPFLAGS_ENABLED, LOG_GROUP_IEM_SYSCALL))
3329	iemLogSyscallProtModeInt(pVCpu, u8Vector, cbInstr);
3330	#endif
3331
3332	/* Check the new EIP against the new CS limit. */
3333	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
3334	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
3335	? Idte.Gate.u16OffsetLow
3336	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
3337	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3338	if (uNewEip > cbLimitCS)
3339	{
3340	Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
3341	u8Vector, uNewEip, cbLimitCS, NewCS));
3342	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3343	}
3344	Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
3345
3346	/* Calc the flag image to push. */
3347	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3348	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3349	fEfl &= ~X86_EFL_RF;
3350	else
3351	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3352
3353	/* From V8086 mode only go to CPL 0. */
3354	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3355	? IEM_GET_CPL(pVCpu) : DescCS.Legacy.Gen.u2Dpl;
3356	if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
3357	{
3358	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
3359	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3360	}
3361
3362	/*
3363	* If the privilege level changes, we need to get a new stack from the TSS.
3364	* This in turns means validating the new SS and ESP...
3365	*/
3366	if (uNewCpl != IEM_GET_CPL(pVCpu))
3367	{
3368	RTSEL NewSS;
3369	uint32_t uNewEsp;
3370	rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
3371	if (rcStrict != VINF_SUCCESS)
3372	return rcStrict;
3373
3374	IEMSELDESC DescSS;
3375	rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
3376	if (rcStrict != VINF_SUCCESS)
3377	return rcStrict;
3378	/* If the new SS is 16-bit, we are only going to use SP, not ESP. */
3379	if (!DescSS.Legacy.Gen.u1DefBig)
3380	{
3381	Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
3382	uNewEsp = (uint16_t)uNewEsp;
3383	}
3384
3385	Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3386
3387	/* Check that there is sufficient space for the stack frame. */
3388	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3389	uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
3390	? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
3391	: (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
3392
3393	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
3394	{
3395	if ( uNewEsp - 1 > cbLimitSS
3396	\|\| uNewEsp < cbStackFrame)
3397	{
3398	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
3399	u8Vector, NewSS, uNewEsp, cbStackFrame));
3400	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3401	}
3402	}
3403	else
3404	{
3405	if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
3406	\|\| uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
3407	{
3408	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
3409	u8Vector, NewSS, uNewEsp, cbStackFrame));
3410	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3411	}
3412	}
3413
3414	/*
3415	* Start making changes.
3416	*/
3417
3418	/* Set the new CPL so that stack accesses use it. */
3419	uint8_t const uOldCpl = IEM_GET_CPL(pVCpu);
3420	IEM_SET_CPL(pVCpu, uNewCpl);
3421
3422	/* Create the stack frame. */
3423	uint8_t bUnmapInfoStackFrame;
3424	RTPTRUNION uStackFrame;
3425	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, &bUnmapInfoStackFrame, cbStackFrame, UINT8_MAX,
3426	uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy),
3427	IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3428	if (rcStrict != VINF_SUCCESS)
3429	return rcStrict;
3430	if (f32BitGate)
3431	{
3432	if (fFlags & IEM_XCPT_FLAGS_ERR)
3433	*uStackFrame.pu32++ = uErr;
3434	uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3435	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3436	uStackFrame.pu32[2] = fEfl;
3437	uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
3438	uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
3439	Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3440	if (fEfl & X86_EFL_VM)
3441	{
3442	uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
3443	uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
3444	uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
3445	uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
3446	uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
3447	}
3448	}
3449	else
3450	{
3451	if (fFlags & IEM_XCPT_FLAGS_ERR)
3452	*uStackFrame.pu16++ = uErr;
3453	uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3454	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3455	uStackFrame.pu16[2] = fEfl;
3456	uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
3457	uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
3458	Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
3459	if (fEfl & X86_EFL_VM)
3460	{
3461	uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
3462	uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
3463	uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
3464	uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
3465	uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
3466	}
3467	}
3468	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoStackFrame);
3469	if (rcStrict != VINF_SUCCESS)
3470	return rcStrict;
3471
3472	/* Mark the selectors 'accessed' (hope this is the correct time). */
3473	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3474	* after pushing the stack frame? (Write protect the gdt + stack to
3475	* find out.) */
3476	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3477	{
3478	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3479	if (rcStrict != VINF_SUCCESS)
3480	return rcStrict;
3481	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3482	}
3483
3484	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3485	{
3486	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
3487	if (rcStrict != VINF_SUCCESS)
3488	return rcStrict;
3489	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3490	}
3491
3492	/*
3493	* Start comitting the register changes (joins with the DPL=CPL branch).
3494	*/
3495	pVCpu->cpum.GstCtx.ss.Sel = NewSS;
3496	pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
3497	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3498	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
3499	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3500	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3501	/** @todo When coming from 32-bit code and operating with a 16-bit TSS and
3502	* 16-bit handler, the high word of ESP remains unchanged (i.e. only
3503	* SP is loaded).
3504	* Need to check the other combinations too:
3505	* - 16-bit TSS, 32-bit handler
3506	* - 32-bit TSS, 16-bit handler */
3507	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3508	pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
3509	else
3510	pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
3511
3512	if (fEfl & X86_EFL_VM)
3513	{
3514	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
3515	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
3516	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
3517	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
3518	}
3519	}
3520	/*
3521	* Same privilege, no stack change and smaller stack frame.
3522	*/
3523	else
3524	{
3525	uint64_t uNewRsp;
3526	uint8_t bUnmapInfoStackFrame;
3527	RTPTRUNION uStackFrame;
3528	uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
3529	rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1,
3530	&uStackFrame.pv, &bUnmapInfoStackFrame, &uNewRsp);
3531	if (rcStrict != VINF_SUCCESS)
3532	return rcStrict;
3533
3534	if (f32BitGate)
3535	{
3536	if (fFlags & IEM_XCPT_FLAGS_ERR)
3537	*uStackFrame.pu32++ = uErr;
3538	uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3539	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| IEM_GET_CPL(pVCpu);
3540	uStackFrame.pu32[2] = fEfl;
3541	}
3542	else
3543	{
3544	if (fFlags & IEM_XCPT_FLAGS_ERR)
3545	*uStackFrame.pu16++ = uErr;
3546	uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3547	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| IEM_GET_CPL(pVCpu);
3548	uStackFrame.pu16[2] = fEfl;
3549	}
3550	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoStackFrame); /* don't use the commit here */
3551	if (rcStrict != VINF_SUCCESS)
3552	return rcStrict;
3553
3554	/* Mark the CS selector as 'accessed'. */
3555	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3556	{
3557	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3558	if (rcStrict != VINF_SUCCESS)
3559	return rcStrict;
3560	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3561	}
3562
3563	/*
3564	* Start committing the register changes (joins with the other branch).
3565	*/
3566	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3567	}
3568
3569	/* ... register committing continues. */
3570	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3571	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3572	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3573	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3574	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3575	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3576
3577	pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
3578	fEfl &= ~fEflToClear;
3579	IEMMISC_SET_EFL(pVCpu, fEfl);
3580
3581	if (fFlags & IEM_XCPT_FLAGS_CR2)
3582	pVCpu->cpum.GstCtx.cr2 = uCr2;
3583
3584	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3585	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3586
3587	/* Make sure the execution flags are correct. */
3588	uint32_t const fExecNew = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
3589	if (fExecNew != pVCpu->iem.s.fExec)
3590	Log(("iemRaiseXcptOrIntInProtMode: fExec %#x -> %#x (xor %#x)\n",
3591	pVCpu->iem.s.fExec, fExecNew, pVCpu->iem.s.fExec ^ fExecNew));
3592	pVCpu->iem.s.fExec = fExecNew;
3593	Assert(IEM_GET_CPL(pVCpu) == uNewCpl);
3594
3595	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3596	}
3597
3598
3599	/**
3600	* Implements exceptions and interrupts for long mode.
3601	*
3602	* @returns VBox strict status code.
3603	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3604	* @param cbInstr The number of bytes to offset rIP by in the return
3605	* address.
3606	* @param u8Vector The interrupt / exception vector number.
3607	* @param fFlags The flags.
3608	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3609	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3610	*/
3611	static VBOXSTRICTRC
3612	iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
3613	uint8_t cbInstr,
3614	uint8_t u8Vector,
3615	uint32_t fFlags,
3616	uint16_t uErr,
3617	uint64_t uCr2) RT_NOEXCEPT
3618	{
3619	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3620
3621	/*
3622	* Read the IDT entry.
3623	*/
3624	uint16_t offIdt = (uint16_t)u8Vector << 4;
3625	if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
3626	{
3627	Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3628	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3629	}
3630	X86DESC64 Idte;
3631	#ifdef _MSC_VER /* Shut up silly compiler warning. */
3632	Idte.au64[0] = 0;
3633	Idte.au64[1] = 0;
3634	#endif
3635	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
3636	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3637	rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
3638	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3639	{
3640	Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3641	return rcStrict;
3642	}
3643	Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
3644	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3645	Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3646
3647	/*
3648	* Check the descriptor type, DPL and such.
3649	* ASSUMES this is done in the same order as described for call-gate calls.
3650	*/
3651	if (Idte.Gate.u1DescType)
3652	{
3653	Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3654	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3655	}
3656	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3657	switch (Idte.Gate.u4Type)
3658	{
3659	case AMD64_SEL_TYPE_SYS_INT_GATE:
3660	fEflToClear \|= X86_EFL_IF;
3661	break;
3662	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
3663	break;
3664
3665	default:
3666	Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3667	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3668	}
3669
3670	/* Check DPL against CPL if applicable. */
3671	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
3672	{
3673	if (IEM_GET_CPL(pVCpu) > Idte.Gate.u2Dpl)
3674	{
3675	Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, IEM_GET_CPL(pVCpu), Idte.Gate.u2Dpl));
3676	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3677	}
3678	}
3679
3680	/* Is it there? */
3681	if (!Idte.Gate.u1Present)
3682	{
3683	Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
3684	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3685	}
3686
3687	/* A null CS is bad. */
3688	RTSEL NewCS = Idte.Gate.u16Sel;
3689	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3690	{
3691	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3692	return iemRaiseGeneralProtectionFault0(pVCpu);
3693	}
3694
3695	/* Fetch the descriptor for the new CS. */
3696	IEMSELDESC DescCS;
3697	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
3698	if (rcStrict != VINF_SUCCESS)
3699	{
3700	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3701	return rcStrict;
3702	}
3703
3704	/* Must be a 64-bit code segment. */
3705	if (!DescCS.Long.Gen.u1DescType)
3706	{
3707	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3708	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3709	}
3710	if ( !DescCS.Long.Gen.u1Long
3711	\|\| DescCS.Long.Gen.u1DefBig
3712	\|\| !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
3713	{
3714	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
3715	u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
3716	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3717	}
3718
3719	/* Don't allow lowering the privilege level. For non-conforming CS
3720	selectors, the CS.DPL sets the privilege level the trap/interrupt
3721	handler runs at. For conforming CS selectors, the CPL remains
3722	unchanged, but the CS.DPL must be <= CPL. */
3723	/** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
3724	* when CPU in Ring-0. Result \#GP? */
3725	if (DescCS.Legacy.Gen.u2Dpl > IEM_GET_CPL(pVCpu))
3726	{
3727	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3728	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, IEM_GET_CPL(pVCpu)));
3729	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3730	}
3731
3732
3733	/* Make sure the selector is present. */
3734	if (!DescCS.Legacy.Gen.u1Present)
3735	{
3736	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3737	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3738	}
3739
3740	/* Check that the new RIP is canonical. */
3741	uint64_t const uNewRip = Idte.Gate.u16OffsetLow
3742	\| ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
3743	\| ((uint64_t)Idte.Gate.u32OffsetTop << 32);
3744	if (!IEM_IS_CANONICAL(uNewRip))
3745	{
3746	Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
3747	return iemRaiseGeneralProtectionFault0(pVCpu);
3748	}
3749
3750	/*
3751	* If the privilege level changes or if the IST isn't zero, we need to get
3752	* a new stack from the TSS.
3753	*/
3754	uint64_t uNewRsp;
3755	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3756	? IEM_GET_CPL(pVCpu) : DescCS.Legacy.Gen.u2Dpl;
3757	if ( uNewCpl != IEM_GET_CPL(pVCpu)
3758	\|\| Idte.Gate.u3IST != 0)
3759	{
3760	rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
3761	if (rcStrict != VINF_SUCCESS)
3762	return rcStrict;
3763	}
3764	else
3765	uNewRsp = pVCpu->cpum.GstCtx.rsp;
3766	uNewRsp &= ~(uint64_t)0xf;
3767
3768	/*
3769	* Calc the flag image to push.
3770	*/
3771	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3772	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3773	fEfl &= ~X86_EFL_RF;
3774	else
3775	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3776
3777	/*
3778	* Start making changes.
3779	*/
3780	/* Set the new CPL so that stack accesses use it. */
3781	uint8_t const uOldCpl = IEM_GET_CPL(pVCpu);
3782	IEM_SET_CPL(pVCpu, uNewCpl);
3783	/** @todo Setting CPL this early seems wrong as it would affect and errors we
3784	* raise accessing the stack and (?) GDT/LDT... */
3785
3786	/* Create the stack frame. */
3787	uint8_t bUnmapInfoStackFrame;
3788	uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
3789	RTPTRUNION uStackFrame;
3790	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, &bUnmapInfoStackFrame, cbStackFrame, UINT8_MAX,
3791	uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3792	if (rcStrict != VINF_SUCCESS)
3793	return rcStrict;
3794
3795	if (fFlags & IEM_XCPT_FLAGS_ERR)
3796	*uStackFrame.pu64++ = uErr;
3797	uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
3798	uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl; /* CPL paranoia */
3799	uStackFrame.pu64[2] = fEfl;
3800	uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
3801	uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
3802	rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfoStackFrame);
3803	if (rcStrict != VINF_SUCCESS)
3804	return rcStrict;
3805
3806	/* Mark the CS selectors 'accessed' (hope this is the correct time). */
3807	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3808	* after pushing the stack frame? (Write protect the gdt + stack to
3809	* find out.) */
3810	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3811	{
3812	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3813	if (rcStrict != VINF_SUCCESS)
3814	return rcStrict;
3815	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3816	}
3817
3818	/*
3819	* Start comitting the register changes.
3820	*/
3821	/** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
3822	* hidden registers when interrupting 32-bit or 16-bit code! */
3823	if (uNewCpl != uOldCpl)
3824	{
3825	pVCpu->cpum.GstCtx.ss.Sel = 0 \| uNewCpl;
3826	pVCpu->cpum.GstCtx.ss.ValidSel = 0 \| uNewCpl;
3827	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3828	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3829	pVCpu->cpum.GstCtx.ss.u64Base = 0;
3830	pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) \| X86DESCATTR_UNUSABLE;
3831	}
3832	pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
3833	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3834	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3835	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3836	pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
3837	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3838	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3839	pVCpu->cpum.GstCtx.rip = uNewRip;
3840
3841	fEfl &= ~fEflToClear;
3842	IEMMISC_SET_EFL(pVCpu, fEfl);
3843
3844	if (fFlags & IEM_XCPT_FLAGS_CR2)
3845	pVCpu->cpum.GstCtx.cr2 = uCr2;
3846
3847	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3848	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3849
3850	iemRecalcExecModeAndCplFlags(pVCpu);
3851
3852	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3853	}
3854
3855
3856	/**
3857	* Implements exceptions and interrupts.
3858	*
3859	* All exceptions and interrupts goes thru this function!
3860	*
3861	* @returns VBox strict status code.
3862	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3863	* @param cbInstr The number of bytes to offset rIP by in the return
3864	* address.
3865	* @param u8Vector The interrupt / exception vector number.
3866	* @param fFlags The flags.
3867	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3868	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3869	*/
3870	VBOXSTRICTRC
3871	iemRaiseXcptOrInt(PVMCPUCC pVCpu,
3872	uint8_t cbInstr,
3873	uint8_t u8Vector,
3874	uint32_t fFlags,
3875	uint16_t uErr,
3876	uint64_t uCr2) RT_NOEXCEPT
3877	{
3878	/*
3879	* Get all the state that we might need here.
3880	*/
3881	IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3882	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3883
3884	#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
3885	/*
3886	* Flush prefetch buffer
3887	*/
3888	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3889	#endif
3890
3891	/*
3892	* Perform the V8086 IOPL check and upgrade the fault without nesting.
3893	*/
3894	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
3895	&& pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
3896	&& (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
3897	\| IEM_XCPT_FLAGS_BP_INSTR
3898	\| IEM_XCPT_FLAGS_ICEBP_INSTR
3899	\| IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
3900	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
3901	{
3902	Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
3903	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3904	u8Vector = X86_XCPT_GP;
3905	uErr = 0;
3906	}
3907
3908	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3909	#ifdef DBGFTRACE_ENABLED
3910	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
3911	pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
3912	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
3913	#endif
3914
3915	/*
3916	* Check if DBGF wants to intercept the exception.
3917	*/
3918	if ( (fFlags & (IEM_XCPT_FLAGS_T_EXT_INT \| IEM_XCPT_FLAGS_T_SOFT_INT))
3919	\|\| !DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + u8Vector)) )
3920	{ /* likely */ }
3921	else
3922	{
3923	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + u8Vector),
3924	DBGFEVENTCTX_INVALID, 1, (uint64_t)uErr);
3925	if (rcStrict != VINF_SUCCESS)
3926	return rcStrict;
3927	}
3928
3929	/*
3930	* Evaluate whether NMI blocking should be in effect.
3931	* Normally, NMI blocking is in effect whenever we inject an NMI.
3932	*/
3933	bool fBlockNmi = u8Vector == X86_XCPT_NMI
3934	&& (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT);
3935
3936	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3937	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3938	{
3939	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
3940	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3941	return rcStrict0;
3942
3943	/* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
3944	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
3945	{
3946	Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
3947	fBlockNmi = false;
3948	}
3949	}
3950	#endif
3951
3952	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3953	if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
3954	{
3955	/*
3956	* If the event is being injected as part of VMRUN, it isn't subject to event
3957	* intercepts in the nested-guest. However, secondary exceptions that occur
3958	* during injection of any event -are- subject to exception intercepts.
3959	*
3960	* See AMD spec. 15.20 "Event Injection".
3961	*/
3962	if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
3963	pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
3964	else
3965	{
3966	/*
3967	* Check and handle if the event being raised is intercepted.
3968	*/
3969	VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3970	if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
3971	return rcStrict0;
3972	}
3973	}
3974	#endif
3975
3976	/*
3977	* Set NMI blocking if necessary.
3978	*/
3979	if (fBlockNmi)
3980	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3981
3982	/*
3983	* Do recursion accounting.
3984	*/
3985	uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
3986	uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
3987	if (pVCpu->iem.s.cXcptRecursions == 0)
3988	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
3989	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
3990	else
3991	{
3992	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
3993	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
3994	pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
3995
3996	if (pVCpu->iem.s.cXcptRecursions >= 4)
3997	{
3998	#ifdef DEBUG_bird
3999	AssertFailed();
4000	#endif
4001	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
4002	}
4003
4004	/*
4005	* Evaluate the sequence of recurring events.
4006	*/
4007	IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
4008	NULL /* pXcptRaiseInfo */);
4009	if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
4010	{ /* likely */ }
4011	else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
4012	{
4013	Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
4014	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
4015	u8Vector = X86_XCPT_DF;
4016	uErr = 0;
4017	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4018	/* VMX nested-guest #DF intercept needs to be checked here. */
4019	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
4020	{
4021	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
4022	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
4023	return rcStrict0;
4024	}
4025	#endif
4026	/* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
4027	if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
4028	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
4029	}
4030	else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
4031	{
4032	Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
4033	return iemInitiateCpuShutdown(pVCpu);
4034	}
4035	else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
4036	{
4037	/* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
4038	Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
4039	if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
4040	&& !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
4041	return VERR_EM_GUEST_CPU_HANG;
4042	}
4043	else
4044	{
4045	AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
4046	enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
4047	return VERR_IEM_IPE_9;
4048	}
4049
4050	/*
4051	* The 'EXT' bit is set when an exception occurs during deliver of an external
4052	* event (such as an interrupt or earlier exception)[1]. Privileged software
4053	* exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
4054	* interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
4055	*
4056	* [1] - Intel spec. 6.13 "Error Code"
4057	* [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
4058	* [3] - Intel Instruction reference for INT n.
4059	*/
4060	if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_T_EXT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR))
4061	&& (fFlags & IEM_XCPT_FLAGS_ERR)
4062	&& u8Vector != X86_XCPT_PF
4063	&& u8Vector != X86_XCPT_DF)
4064	{
4065	uErr \|= X86_TRAP_ERR_EXTERNAL;
4066	}
4067	}
4068
4069	pVCpu->iem.s.cXcptRecursions++;
4070	pVCpu->iem.s.uCurXcpt = u8Vector;
4071	pVCpu->iem.s.fCurXcpt = fFlags;
4072	pVCpu->iem.s.uCurXcptErr = uErr;
4073	pVCpu->iem.s.uCurXcptCr2 = uCr2;
4074
4075	/*
4076	* Extensive logging.
4077	*/
4078	#if defined(LOG_ENABLED) && defined(IN_RING3)
4079	if (LogIs3Enabled())
4080	{
4081	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
4082	char szRegs[4096];
4083	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4084	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4085	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4086	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4087	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4088	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4089	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4090	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4091	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4092	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4093	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4094	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4095	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4096	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4097	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4098	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4099	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4100	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4101	" efer=%016VR{efer}\n"
4102	" pat=%016VR{pat}\n"
4103	" sf_mask=%016VR{sf_mask}\n"
4104	"krnl_gs_base=%016VR{krnl_gs_base}\n"
4105	" lstar=%016VR{lstar}\n"
4106	" star=%016VR{star} cstar=%016VR{cstar}\n"
4107	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4108	);
4109
4110	char szInstr[256];
4111	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4112	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
4113	szInstr, sizeof(szInstr), NULL);
4114	Log3(("%s%s\n", szRegs, szInstr));
4115	}
4116	#endif /* LOG_ENABLED */
4117
4118	/*
4119	* Stats.
4120	*/
4121	uint64_t const uTimestamp = ASMReadTSC();
4122	if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
4123	{
4124	STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; });
4125	EMHistoryAddExit(pVCpu,
4126	fFlags & IEM_XCPT_FLAGS_T_EXT_INT
4127	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_IEM, u8Vector)
4128	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_IEM, u8Vector \| 0x100),
4129	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, uTimestamp);
4130	}
4131	else
4132	{
4133	if (u8Vector < RT_ELEMENTS(pVCpu->iem.s.aStatXcpts))
4134	STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]);
4135	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector),
4136	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, uTimestamp);
4137	if (fFlags & IEM_XCPT_FLAGS_ERR)
4138	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector \| EMEXIT_F_XCPT_ERRCD), uErr, uTimestamp);
4139	if (fFlags & IEM_XCPT_FLAGS_CR2)
4140	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector \| EMEXIT_F_XCPT_CR2), uCr2, uTimestamp);
4141	}
4142
4143	/*
4144	* #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3)
4145	* to ensure that a stale TLB or paging cache entry will only cause one
4146	* spurious #PF.
4147	*/
4148	if ( u8Vector == X86_XCPT_PF
4149	&& (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2))
4150	IEMTlbInvalidatePage(pVCpu, uCr2);
4151
4152	/*
4153	* Call the mode specific worker function.
4154	*/
4155	VBOXSTRICTRC rcStrict;
4156	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
4157	rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
4158	else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
4159	rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
4160	else
4161	rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
4162
4163	/* Flush the prefetch buffer. */
4164	iemOpcodeFlushHeavy(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
4165
4166	/*
4167	* Unwind.
4168	*/
4169	pVCpu->iem.s.cXcptRecursions--;
4170	pVCpu->iem.s.uCurXcpt = uPrevXcpt;
4171	pVCpu->iem.s.fCurXcpt = fPrevXcpt;
4172	Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
4173	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel,
4174	pVCpu->cpum.GstCtx.esp, IEM_GET_CPL(pVCpu), pVCpu->iem.s.cXcptRecursions + 1));
4175	return rcStrict;
4176	}
4177
4178	#ifdef IEM_WITH_SETJMP
4179	/**
4180	* See iemRaiseXcptOrInt. Will not return.
4181	*/
4182	DECL_NO_RETURN(void)
4183	iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
4184	uint8_t cbInstr,
4185	uint8_t u8Vector,
4186	uint32_t fFlags,
4187	uint16_t uErr,
4188	uint64_t uCr2) IEM_NOEXCEPT_MAY_LONGJMP
4189	{
4190	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
4191	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
4192	}
4193	#endif
4194
4195
4196	/** \#DE - 00. */
4197	VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT
4198	{
4199	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4200	}
4201
4202
4203	#ifdef IEM_WITH_SETJMP
4204	/** \#DE - 00. */
4205	DECL_NO_RETURN(void) iemRaiseDivideErrorJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4206	{
4207	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4208	}
4209	#endif
4210
4211
4212	/** \#DB - 01.
4213	* @note This automatically clear DR7.GD. */
4214	VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT
4215	{
4216	/* This always clears RF (via IEM_XCPT_FLAGS_DRx_INSTR_BP). */
4217	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
4218	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_DRx_INSTR_BP, 0, 0);
4219	}
4220
4221
4222	/** \#BR - 05. */
4223	VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT
4224	{
4225	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4226	}
4227
4228
4229	/** \#UD - 06. */
4230	VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT
4231	{
4232	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4233	}
4234
4235
4236	#ifdef IEM_WITH_SETJMP
4237	/** \#UD - 06. */
4238	DECL_NO_RETURN(void) iemRaiseUndefinedOpcodeJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4239	{
4240	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4241	}
4242	#endif
4243
4244
4245	/** \#NM - 07. */
4246	VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT
4247	{
4248	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4249	}
4250
4251
4252	#ifdef IEM_WITH_SETJMP
4253	/** \#NM - 07. */
4254	DECL_NO_RETURN(void) iemRaiseDeviceNotAvailableJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4255	{
4256	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4257	}
4258	#endif
4259
4260
4261	/** \#TS(err) - 0a. */
4262	VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4263	{
4264	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4265	}
4266
4267
4268	/** \#TS(tr) - 0a. */
4269	VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT
4270	{
4271	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4272	pVCpu->cpum.GstCtx.tr.Sel, 0);
4273	}
4274
4275
4276	/** \#TS(0) - 0a. */
4277	VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
4278	{
4279	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4280	0, 0);
4281	}
4282
4283
4284	/** \#TS(err) - 0a. */
4285	VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4286	{
4287	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4288	uSel & X86_SEL_MASK_OFF_RPL, 0);
4289	}
4290
4291
4292	/** \#NP(err) - 0b. */
4293	VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4294	{
4295	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4296	}
4297
4298
4299	/** \#NP(sel) - 0b. */
4300	VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4301	{
4302	Log(("iemRaiseSelectorNotPresentBySelector: cs:rip=%04x:%RX64 uSel=%#x\n",
4303	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uSel));
4304	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4305	uSel & ~X86_SEL_RPL, 0);
4306	}
4307
4308
4309	/** \#SS(seg) - 0c. */
4310	VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4311	{
4312	Log(("iemRaiseStackSelectorNotPresentBySelector: cs:rip=%04x:%RX64 uSel=%#x\n",
4313	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uSel));
4314	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4315	uSel & ~X86_SEL_RPL, 0);
4316	}
4317
4318
4319	/** \#SS(err) - 0c. */
4320	VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4321	{
4322	Log(("iemRaiseStackSelectorNotPresentWithErr: cs:rip=%04x:%RX64 uErr=%#x\n",
4323	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uErr));
4324	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4325	}
4326
4327
4328	/** \#GP(n) - 0d. */
4329	VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4330	{
4331	Log(("iemRaiseGeneralProtectionFault: cs:rip=%04x:%RX64 uErr=%#x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uErr));
4332	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4333	}
4334
4335
4336	/** \#GP(0) - 0d. */
4337	VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
4338	{
4339	Log(("iemRaiseGeneralProtectionFault0: cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
4340	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4341	}
4342
4343	#ifdef IEM_WITH_SETJMP
4344	/** \#GP(0) - 0d. */
4345	DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4346	{
4347	Log(("iemRaiseGeneralProtectionFault0Jmp: cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
4348	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4349	}
4350	#endif
4351
4352
4353	/** \#GP(sel) - 0d. */
4354	VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4355	{
4356	Log(("iemRaiseGeneralProtectionFaultBySelector: cs:rip=%04x:%RX64 Sel=%#x\n",
4357	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, Sel));
4358	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4359	Sel & ~X86_SEL_RPL, 0);
4360	}
4361
4362
4363	/** \#GP(0) - 0d. */
4364	VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT
4365	{
4366	Log(("iemRaiseNotCanonical: cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
4367	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4368	}
4369
4370
4371	/** \#GP(sel) - 0d. */
4372	VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4373	{
4374	Log(("iemRaiseSelectorBounds: cs:rip=%04x:%RX64 iSegReg=%d fAccess=%#x\n",
4375	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iSegReg, fAccess));
4376	NOREF(iSegReg); NOREF(fAccess);
4377	return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4378	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4379	}
4380
4381	#ifdef IEM_WITH_SETJMP
4382	/** \#GP(sel) - 0d, longjmp. */
4383	DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP
4384	{
4385	Log(("iemRaiseSelectorBoundsJmp: cs:rip=%04x:%RX64 iSegReg=%d fAccess=%#x\n",
4386	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iSegReg, fAccess));
4387	NOREF(iSegReg); NOREF(fAccess);
4388	iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4389	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4390	}
4391	#endif
4392
4393	/** \#GP(sel) - 0d. */
4394	VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4395	{
4396	Log(("iemRaiseSelectorBoundsBySelector: cs:rip=%04x:%RX64 Sel=%#x\n",
4397	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, Sel));
4398	NOREF(Sel);
4399	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4400	}
4401
4402	#ifdef IEM_WITH_SETJMP
4403	/** \#GP(sel) - 0d, longjmp. */
4404	DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) IEM_NOEXCEPT_MAY_LONGJMP
4405	{
4406	Log(("iemRaiseSelectorBoundsBySelectorJmp: cs:rip=%04x:%RX64 Sel=%#x\n",
4407	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, Sel));
4408	NOREF(Sel);
4409	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4410	}
4411	#endif
4412
4413
4414	/** \#GP(sel) - 0d. */
4415	VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4416	{
4417	Log(("iemRaiseSelectorInvalidAccess: cs:rip=%04x:%RX64 iSegReg=%d fAccess=%#x\n",
4418	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iSegReg, fAccess));
4419	NOREF(iSegReg); NOREF(fAccess);
4420	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4421	}
4422
4423	#ifdef IEM_WITH_SETJMP
4424	/** \#GP(sel) - 0d, longjmp. */
4425	DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP
4426	{
4427	NOREF(iSegReg); NOREF(fAccess);
4428	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4429	}
4430	#endif
4431
4432
4433	/** \#PF(n) - 0e. */
4434	VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t cbAccess, uint32_t fAccess, int rc) RT_NOEXCEPT
4435	{
4436	uint16_t uErr;
4437	switch (rc)
4438	{
4439	case VERR_PAGE_NOT_PRESENT:
4440	case VERR_PAGE_TABLE_NOT_PRESENT:
4441	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
4442	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
4443	uErr = 0;
4444	break;
4445
4446	default:
4447	AssertMsgFailed(("%Rrc\n", rc));
4448	RT_FALL_THRU();
4449	case VERR_ACCESS_DENIED:
4450	uErr = X86_TRAP_PF_P;
4451	break;
4452
4453	/** @todo reserved */
4454	}
4455
4456	if (IEM_GET_CPL(pVCpu) == 3)
4457	uErr \|= X86_TRAP_PF_US;
4458
4459	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
4460	&& ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
4461	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
4462	uErr \|= X86_TRAP_PF_ID;
4463
4464	#if 0 /* This is so much non-sense, really. Why was it done like that? */
4465	/* Note! RW access callers reporting a WRITE protection fault, will clear
4466	the READ flag before calling. So, read-modify-write accesses (RW)
4467	can safely be reported as READ faults. */
4468	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
4469	uErr \|= X86_TRAP_PF_RW;
4470	#else
4471	if (fAccess & IEM_ACCESS_TYPE_WRITE)
4472	{
4473	/// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
4474	/// (regardless of outcome of the comparison in the latter case).
4475	//if (!(fAccess & IEM_ACCESS_TYPE_READ))
4476	uErr \|= X86_TRAP_PF_RW;
4477	}
4478	#endif
4479
4480	/* For FXSAVE and FRSTOR the #PF is typically reported at the max address
4481	of the memory operand rather than at the start of it. (Not sure what
4482	happens if it crosses a page boundrary.) The current heuristics for
4483	this is to report the #PF for the last byte if the access is more than
4484	64 bytes. This is probably not correct, but we can work that out later,
4485	main objective now is to get FXSAVE to work like for real hardware and
4486	make bs3-cpu-basic2 work. */
4487	if (cbAccess <= 64)
4488	{ /* likely*/ }
4489	else
4490	GCPtrWhere += cbAccess - 1;
4491
4492	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
4493	uErr, GCPtrWhere);
4494	}
4495
4496	#ifdef IEM_WITH_SETJMP
4497	/** \#PF(n) - 0e, longjmp. */
4498	DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t cbAccess,
4499	uint32_t fAccess, int rc) IEM_NOEXCEPT_MAY_LONGJMP
4500	{
4501	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, cbAccess, fAccess, rc)));
4502	}
4503	#endif
4504
4505
4506	/** \#MF(0) - 10. */
4507	VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu) RT_NOEXCEPT
4508	{
4509	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE)
4510	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4511
4512	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
4513	PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
4514	return iemRegUpdateRipAndFinishClearingRF(pVCpu);
4515	}
4516
4517	#ifdef IEM_WITH_SETJMP
4518	/** \#MF(0) - 10, longjmp. */
4519	DECL_NO_RETURN(void) iemRaiseMathFaultJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4520	{
4521	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaiseMathFault(pVCpu)));
4522	}
4523	#endif
4524
4525
4526	/** \#AC(0) - 11. */
4527	VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu) RT_NOEXCEPT
4528	{
4529	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4530	}
4531
4532	#ifdef IEM_WITH_SETJMP
4533	/** \#AC(0) - 11, longjmp. */
4534	DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4535	{
4536	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu)));
4537	}
4538	#endif
4539
4540
4541	/** \#XF(0)/\#XM(0) - 19. */
4542	VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT
4543	{
4544	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4545	}
4546
4547
4548	#ifdef IEM_WITH_SETJMP
4549	/** \#XF(0)/\#XM(0) - 19s, longjmp. */
4550	DECL_NO_RETURN(void) iemRaiseSimdFpExceptionJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
4551	{
4552	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaiseSimdFpException(pVCpu)));
4553	}
4554	#endif
4555
4556
4557	/** Accessed via IEMOP_RAISE_DIVIDE_ERROR. */
4558	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
4559	{
4560	NOREF(cbInstr);
4561	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4562	}
4563
4564
4565	/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */
4566	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
4567	{
4568	NOREF(cbInstr);
4569	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4570	}
4571
4572
4573	/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */
4574	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
4575	{
4576	NOREF(cbInstr);
4577	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4578	}
4579
4580
4581	/** @} */
4582
4583	/** @name Common opcode decoders.
4584	* @{
4585	*/
4586	//#include <iprt/mem.h>
4587
4588	/**
4589	* Used to add extra details about a stub case.
4590	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4591	*/
4592	void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT
4593	{
4594	#if defined(LOG_ENABLED) && defined(IN_RING3)
4595	PVM pVM = pVCpu->CTX_SUFF(pVM);
4596	char szRegs[4096];
4597	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4598	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4599	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4600	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4601	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4602	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4603	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4604	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4605	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4606	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4607	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4608	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4609	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4610	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4611	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4612	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4613	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4614	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4615	" efer=%016VR{efer}\n"
4616	" pat=%016VR{pat}\n"
4617	" sf_mask=%016VR{sf_mask}\n"
4618	"krnl_gs_base=%016VR{krnl_gs_base}\n"
4619	" lstar=%016VR{lstar}\n"
4620	" star=%016VR{star} cstar=%016VR{cstar}\n"
4621	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4622	);
4623
4624	char szInstr[256];
4625	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4626	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
4627	szInstr, sizeof(szInstr), NULL);
4628
4629	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
4630	#else
4631	RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
4632	#endif
4633	}
4634
4635	/** @} */
4636
4637
4638
4639	/** @name Register Access.
4640	* @{
4641	*/
4642
4643	/**
4644	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
4645	*
4646	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4647	* segment limit.
4648	*
4649	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4650	* @param cbInstr Instruction size.
4651	* @param offNextInstr The offset of the next instruction.
4652	* @param enmEffOpSize Effective operand size.
4653	*/
4654	VBOXSTRICTRC iemRegRipRelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
4655	IEMMODE enmEffOpSize) RT_NOEXCEPT
4656	{
4657	switch (enmEffOpSize)
4658	{
4659	case IEMMODE_16BIT:
4660	{
4661	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
4662	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
4663	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */))
4664	pVCpu->cpum.GstCtx.rip = uNewIp;
4665	else
4666	return iemRaiseGeneralProtectionFault0(pVCpu);
4667	break;
4668	}
4669
4670	case IEMMODE_32BIT:
4671	{
4672	Assert(!IEM_IS_64BIT_CODE(pVCpu));
4673	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4674
4675	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
4676	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
4677	pVCpu->cpum.GstCtx.rip = uNewEip;
4678	else
4679	return iemRaiseGeneralProtectionFault0(pVCpu);
4680	break;
4681	}
4682
4683	case IEMMODE_64BIT:
4684	{
4685	Assert(IEM_IS_64BIT_CODE(pVCpu));
4686
4687	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
4688	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
4689	pVCpu->cpum.GstCtx.rip = uNewRip;
4690	else
4691	return iemRaiseGeneralProtectionFault0(pVCpu);
4692	break;
4693	}
4694
4695	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4696	}
4697
4698	#ifndef IEM_WITH_CODE_TLB
4699	/* Flush the prefetch buffer. */
4700	pVCpu->iem.s.cbOpcode = cbInstr;
4701	#endif
4702
4703	/*
4704	* Clear RF and finish the instruction (maybe raise #DB).
4705	*/
4706	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
4707	}
4708
4709
4710	/**
4711	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
4712	*
4713	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4714	* segment limit.
4715	*
4716	* @returns Strict VBox status code.
4717	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4718	* @param cbInstr Instruction size.
4719	* @param offNextInstr The offset of the next instruction.
4720	*/
4721	VBOXSTRICTRC iemRegRipRelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offNextInstr) RT_NOEXCEPT
4722	{
4723	Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
4724
4725	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
4726	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
4727	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checking in 64-bit mode */))
4728	pVCpu->cpum.GstCtx.rip = uNewIp;
4729	else
4730	return iemRaiseGeneralProtectionFault0(pVCpu);
4731
4732	#ifndef IEM_WITH_CODE_TLB
4733	/* Flush the prefetch buffer. */
4734	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4735	#endif
4736
4737	/*
4738	* Clear RF and finish the instruction (maybe raise #DB).
4739	*/
4740	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
4741	}
4742
4743
4744	/**
4745	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
4746	*
4747	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4748	* segment limit.
4749	*
4750	* @returns Strict VBox status code.
4751	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4752	* @param cbInstr Instruction size.
4753	* @param offNextInstr The offset of the next instruction.
4754	* @param enmEffOpSize Effective operand size.
4755	*/
4756	VBOXSTRICTRC iemRegRipRelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr,
4757	IEMMODE enmEffOpSize) RT_NOEXCEPT
4758	{
4759	if (enmEffOpSize == IEMMODE_32BIT)
4760	{
4761	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));
4762
4763	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
4764	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
4765	pVCpu->cpum.GstCtx.rip = uNewEip;
4766	else
4767	return iemRaiseGeneralProtectionFault0(pVCpu);
4768	}
4769	else
4770	{
4771	Assert(enmEffOpSize == IEMMODE_64BIT);
4772
4773	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
4774	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
4775	pVCpu->cpum.GstCtx.rip = uNewRip;
4776	else
4777	return iemRaiseGeneralProtectionFault0(pVCpu);
4778	}
4779
4780	#ifndef IEM_WITH_CODE_TLB
4781	/* Flush the prefetch buffer. */
4782	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4783	#endif
4784
4785	/*
4786	* Clear RF and finish the instruction (maybe raise #DB).
4787	*/
4788	return iemRegFinishClearingRF(pVCpu, VINF_SUCCESS);
4789	}
4790
4791	/** @} */
4792
4793
4794	/** @name FPU access and helpers.
4795	*
4796	* @{
4797	*/
4798
4799	/**
4800	* Updates the x87.DS and FPUDP registers.
4801	*
4802	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4803	* @param pFpuCtx The FPU context.
4804	* @param iEffSeg The effective segment register.
4805	* @param GCPtrEff The effective address relative to @a iEffSeg.
4806	*/
4807	DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
4808	{
4809	RTSEL sel;
4810	switch (iEffSeg)
4811	{
4812	case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
4813	case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
4814	case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
4815	case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
4816	case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
4817	case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
4818	default:
4819	AssertMsgFailed(("%d\n", iEffSeg));
4820	sel = pVCpu->cpum.GstCtx.ds.Sel;
4821	}
4822	/** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
4823	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4824	{
4825	pFpuCtx->DS = 0;
4826	pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
4827	}
4828	else if (!IEM_IS_LONG_MODE(pVCpu)) /** @todo this is weird. explain. */
4829	{
4830	pFpuCtx->DS = sel;
4831	pFpuCtx->FPUDP = GCPtrEff;
4832	}
4833	else
4834	(uint64_t )&pFpuCtx->FPUDP = GCPtrEff;
4835	}
4836
4837
4838	/**
4839	* Rotates the stack registers in the push direction.
4840	*
4841	* @param pFpuCtx The FPU context.
4842	* @remarks This is a complete waste of time, but fxsave stores the registers in
4843	* stack order.
4844	*/
4845	DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
4846	{
4847	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
4848	pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
4849	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
4850	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
4851	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
4852	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
4853	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
4854	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
4855	pFpuCtx->aRegs[0].r80 = r80Tmp;
4856	}
4857
4858
4859	/**
4860	* Rotates the stack registers in the pop direction.
4861	*
4862	* @param pFpuCtx The FPU context.
4863	* @remarks This is a complete waste of time, but fxsave stores the registers in
4864	* stack order.
4865	*/
4866	DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
4867	{
4868	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
4869	pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
4870	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
4871	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
4872	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
4873	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
4874	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
4875	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
4876	pFpuCtx->aRegs[7].r80 = r80Tmp;
4877	}
4878
4879
4880	/**
4881	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
4882	* exception prevents it.
4883	*
4884	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4885	* @param pResult The FPU operation result to push.
4886	* @param pFpuCtx The FPU context.
4887	*/
4888	static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4889	{
4890	/* Update FSW and bail if there are pending exceptions afterwards. */
4891	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4892	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4893	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4894	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4895	{
4896	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES))
4897	Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n",
4898	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4899	pFpuCtx->FSW = fFsw;
4900	return;
4901	}
4902
4903	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4904	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4905	{
4906	/* All is fine, push the actual value. */
4907	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4908	pFpuCtx->aRegs[7].r80 = pResult->r80Result;
4909	}
4910	else if (pFpuCtx->FCW & X86_FCW_IM)
4911	{
4912	/* Masked stack overflow, push QNaN. */
4913	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4914	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4915	}
4916	else
4917	{
4918	/* Raise stack overflow, don't push anything. */
4919	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4920	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4921	Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4922	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4923	return;
4924	}
4925
4926	fFsw &= ~X86_FSW_TOP_MASK;
4927	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4928	pFpuCtx->FSW = fFsw;
4929
4930	iemFpuRotateStackPush(pFpuCtx);
4931	RT_NOREF(pVCpu);
4932	}
4933
4934
4935	/**
4936	* Stores a result in a FPU register and updates the FSW and FTW.
4937	*
4938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4939	* @param pFpuCtx The FPU context.
4940	* @param pResult The result to store.
4941	* @param iStReg Which FPU register to store it in.
4942	*/
4943	static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4944	{
4945	Assert(iStReg < 8);
4946	uint16_t fNewFsw = pFpuCtx->FSW;
4947	uint16_t const iReg = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK;
4948	fNewFsw &= ~X86_FSW_C_MASK;
4949	fNewFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4950	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4951	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4952	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4953	pFpuCtx->FSW = fNewFsw;
4954	pFpuCtx->FTW \|= RT_BIT(iReg);
4955	pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
4956	RT_NOREF(pVCpu);
4957	}
4958
4959
4960	/**
4961	* Only updates the FPU status word (FSW) with the result of the current
4962	* instruction.
4963	*
4964	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4965	* @param pFpuCtx The FPU context.
4966	* @param u16FSW The FSW output of the current instruction.
4967	*/
4968	static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT
4969	{
4970	uint16_t fNewFsw = pFpuCtx->FSW;
4971	fNewFsw &= ~X86_FSW_C_MASK;
4972	fNewFsw \|= u16FSW & ~X86_FSW_TOP_MASK;
4973	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4974	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4975	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4976	pFpuCtx->FSW = fNewFsw;
4977	RT_NOREF(pVCpu);
4978	}
4979
4980
4981	/**
4982	* Pops one item off the FPU stack if no pending exception prevents it.
4983	*
4984	* @param pFpuCtx The FPU context.
4985	*/
4986	static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4987	{
4988	/* Check pending exceptions. */
4989	uint16_t uFSW = pFpuCtx->FSW;
4990	if ( (pFpuCtx->FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4991	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4992	return;
4993
4994	/* TOP--. */
4995	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
4996	uFSW &= ~X86_FSW_TOP_MASK;
4997	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
4998	pFpuCtx->FSW = uFSW;
4999
5000	/* Mark the previous ST0 as empty. */
5001	iOldTop >>= X86_FSW_TOP_SHIFT;
5002	pFpuCtx->FTW &= ~RT_BIT(iOldTop);
5003
5004	/* Rotate the registers. */
5005	iemFpuRotateStackPop(pFpuCtx);
5006	}
5007
5008
5009	/**
5010	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
5011	*
5012	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5013	* @param pResult The FPU operation result to push.
5014	* @param uFpuOpcode The FPU opcode value.
5015	*/
5016	void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint16_t uFpuOpcode) RT_NOEXCEPT
5017	{
5018	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5019	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5020	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
5021	}
5022
5023
5024	/**
5025	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
5026	* and sets FPUDP and FPUDS.
5027	*
5028	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5029	* @param pResult The FPU operation result to push.
5030	* @param iEffSeg The effective segment register.
5031	* @param GCPtrEff The effective address relative to @a iEffSeg.
5032	* @param uFpuOpcode The FPU opcode value.
5033	*/
5034	void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff,
5035	uint16_t uFpuOpcode) RT_NOEXCEPT
5036	{
5037	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5038	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5039	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5040	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
5041	}
5042
5043
5044	/**
5045	* Replace ST0 with the first value and push the second onto the FPU stack,
5046	* unless a pending exception prevents it.
5047	*
5048	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5049	* @param pResult The FPU operation result to store and push.
5050	* @param uFpuOpcode The FPU opcode value.
5051	*/
5052	void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult, uint16_t uFpuOpcode) RT_NOEXCEPT
5053	{
5054	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5055	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5056
5057	/* Update FSW and bail if there are pending exceptions afterwards. */
5058	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
5059	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
5060	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
5061	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
5062	{
5063	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
5064	Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n",
5065	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
5066	pFpuCtx->FSW = fFsw;
5067	return;
5068	}
5069
5070	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
5071	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
5072	{
5073	/* All is fine, push the actual value. */
5074	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5075	pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
5076	pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
5077	}
5078	else if (pFpuCtx->FCW & X86_FCW_IM)
5079	{
5080	/* Masked stack overflow, push QNaN. */
5081	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
5082	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
5083	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5084	}
5085	else
5086	{
5087	/* Raise stack overflow, don't push anything. */
5088	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
5089	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
5090	Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n",
5091	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5092	return;
5093	}
5094
5095	fFsw &= ~X86_FSW_TOP_MASK;
5096	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
5097	pFpuCtx->FSW = fFsw;
5098
5099	iemFpuRotateStackPush(pFpuCtx);
5100	}
5101
5102
5103	/**
5104	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
5105	* FOP.
5106	*
5107	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5108	* @param pResult The result to store.
5109	* @param iStReg Which FPU register to store it in.
5110	* @param uFpuOpcode The FPU opcode value.
5111	*/
5112	void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg, uint16_t uFpuOpcode) RT_NOEXCEPT
5113	{
5114	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5115	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5116	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
5117	}
5118
5119
5120	/**
5121	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
5122	* FOP, and then pops the stack.
5123	*
5124	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5125	* @param pResult The result to store.
5126	* @param iStReg Which FPU register to store it in.
5127	* @param uFpuOpcode The FPU opcode value.
5128	*/
5129	void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg, uint16_t uFpuOpcode) RT_NOEXCEPT
5130	{
5131	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5132	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5133	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
5134	iemFpuMaybePopOne(pFpuCtx);
5135	}
5136
5137
5138	/**
5139	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
5140	* FPUDP, and FPUDS.
5141	*
5142	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5143	* @param pResult The result to store.
5144	* @param iStReg Which FPU register to store it in.
5145	* @param iEffSeg The effective memory operand selector register.
5146	* @param GCPtrEff The effective memory operand offset.
5147	* @param uFpuOpcode The FPU opcode value.
5148	*/
5149	void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
5150	uint8_t iEffSeg, RTGCPTR GCPtrEff, uint16_t uFpuOpcode) RT_NOEXCEPT
5151	{
5152	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5153	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5154	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5155	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
5156	}
5157
5158
5159	/**
5160	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
5161	* FPUDP, and FPUDS, and then pops the stack.
5162	*
5163	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5164	* @param pResult The result to store.
5165	* @param iStReg Which FPU register to store it in.
5166	* @param iEffSeg The effective memory operand selector register.
5167	* @param GCPtrEff The effective memory operand offset.
5168	* @param uFpuOpcode The FPU opcode value.
5169	*/
5170	void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
5171	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff, uint16_t uFpuOpcode) RT_NOEXCEPT
5172	{
5173	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5174	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5175	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5176	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
5177	iemFpuMaybePopOne(pFpuCtx);
5178	}
5179
5180
5181	/**
5182	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
5183	*
5184	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5185	* @param uFpuOpcode The FPU opcode value.
5186	*/
5187	void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu, uint16_t uFpuOpcode) RT_NOEXCEPT
5188	{
5189	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5190	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5191	}
5192
5193
5194	/**
5195	* Updates the FSW, FOP, FPUIP, and FPUCS.
5196	*
5197	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5198	* @param u16FSW The FSW from the current instruction.
5199	* @param uFpuOpcode The FPU opcode value.
5200	*/
5201	void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW, uint16_t uFpuOpcode) RT_NOEXCEPT
5202	{
5203	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5204	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5205	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
5206	}
5207
5208
5209	/**
5210	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
5211	*
5212	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5213	* @param u16FSW The FSW from the current instruction.
5214	* @param uFpuOpcode The FPU opcode value.
5215	*/
5216	void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint16_t uFpuOpcode) RT_NOEXCEPT
5217	{
5218	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5219	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5220	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
5221	iemFpuMaybePopOne(pFpuCtx);
5222	}
5223
5224
5225	/**
5226	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
5227	*
5228	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5229	* @param u16FSW The FSW from the current instruction.
5230	* @param iEffSeg The effective memory operand selector register.
5231	* @param GCPtrEff The effective memory operand offset.
5232	* @param uFpuOpcode The FPU opcode value.
5233	*/
5234	void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff, uint16_t uFpuOpcode) RT_NOEXCEPT
5235	{
5236	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5237	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5238	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5239	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
5240	}
5241
5242
5243	/**
5244	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
5245	*
5246	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5247	* @param u16FSW The FSW from the current instruction.
5248	* @param uFpuOpcode The FPU opcode value.
5249	*/
5250	void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint16_t uFpuOpcode) RT_NOEXCEPT
5251	{
5252	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5253	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5254	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
5255	iemFpuMaybePopOne(pFpuCtx);
5256	iemFpuMaybePopOne(pFpuCtx);
5257	}
5258
5259
5260	/**
5261	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
5262	*
5263	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5264	* @param u16FSW The FSW from the current instruction.
5265	* @param iEffSeg The effective memory operand selector register.
5266	* @param GCPtrEff The effective memory operand offset.
5267	* @param uFpuOpcode The FPU opcode value.
5268	*/
5269	void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff,
5270	uint16_t uFpuOpcode) RT_NOEXCEPT
5271	{
5272	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5273	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5274	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5275	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
5276	iemFpuMaybePopOne(pFpuCtx);
5277	}
5278
5279
5280	/**
5281	* Worker routine for raising an FPU stack underflow exception.
5282	*
5283	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5284	* @param pFpuCtx The FPU context.
5285	* @param iStReg The stack register being accessed.
5286	*/
5287	static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg)
5288	{
5289	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
5290	if (pFpuCtx->FCW & X86_FCW_IM)
5291	{
5292	/* Masked underflow. */
5293	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5294	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5295	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
5296	if (iStReg != UINT8_MAX)
5297	{
5298	pFpuCtx->FTW \|= RT_BIT(iReg);
5299	iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
5300	}
5301	}
5302	else
5303	{
5304	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5305	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5306	Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n",
5307	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5308	}
5309	RT_NOREF(pVCpu);
5310	}
5311
5312
5313	/**
5314	* Raises a FPU stack underflow exception.
5315	*
5316	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5317	* @param iStReg The destination register that should be loaded
5318	* with QNaN if \#IS is not masked. Specify
5319	* UINT8_MAX if none (like for fcom).
5320	* @param uFpuOpcode The FPU opcode value.
5321	*/
5322	void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg, uint16_t uFpuOpcode) RT_NOEXCEPT
5323	{
5324	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5325	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5326	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5327	}
5328
5329
5330	void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff, uint16_t uFpuOpcode) RT_NOEXCEPT
5331	{
5332	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5333	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5334	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5335	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5336	}
5337
5338
5339	void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint16_t uFpuOpcode) RT_NOEXCEPT
5340	{
5341	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5342	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5343	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5344	iemFpuMaybePopOne(pFpuCtx);
5345	}
5346
5347
5348	void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff,
5349	uint16_t uFpuOpcode) RT_NOEXCEPT
5350	{
5351	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5352	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5353	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5354	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5355	iemFpuMaybePopOne(pFpuCtx);
5356	}
5357
5358
5359	void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu, uint16_t uFpuOpcode) RT_NOEXCEPT
5360	{
5361	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5362	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5363	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX);
5364	iemFpuMaybePopOne(pFpuCtx);
5365	iemFpuMaybePopOne(pFpuCtx);
5366	}
5367
5368
5369	void iemFpuStackPushUnderflow(PVMCPUCC pVCpu, uint16_t uFpuOpcode) RT_NOEXCEPT
5370	{
5371	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5372	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5373
5374	if (pFpuCtx->FCW & X86_FCW_IM)
5375	{
5376	/* Masked overflow - Push QNaN. */
5377	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5378	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5379	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5380	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5381	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5382	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5383	iemFpuRotateStackPush(pFpuCtx);
5384	}
5385	else
5386	{
5387	/* Exception pending - don't change TOP or the register stack. */
5388	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5389	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5390	Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n",
5391	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5392	}
5393	}
5394
5395
5396	void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu, uint16_t uFpuOpcode) RT_NOEXCEPT
5397	{
5398	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5399	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5400
5401	if (pFpuCtx->FCW & X86_FCW_IM)
5402	{
5403	/* Masked overflow - Push QNaN. */
5404	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5405	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5406	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5407	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5408	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5409	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
5410	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5411	iemFpuRotateStackPush(pFpuCtx);
5412	}
5413	else
5414	{
5415	/* Exception pending - don't change TOP or the register stack. */
5416	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5417	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5418	Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n",
5419	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5420	}
5421	}
5422
5423
5424	/**
5425	* Worker routine for raising an FPU stack overflow exception on a push.
5426	*
5427	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5428	* @param pFpuCtx The FPU context.
5429	*/
5430	static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
5431	{
5432	if (pFpuCtx->FCW & X86_FCW_IM)
5433	{
5434	/* Masked overflow. */
5435	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5436	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5437	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
5438	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5439	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5440	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5441	iemFpuRotateStackPush(pFpuCtx);
5442	}
5443	else
5444	{
5445	/* Exception pending - don't change TOP or the register stack. */
5446	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5447	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5448	Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n",
5449	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5450	}
5451	RT_NOREF(pVCpu);
5452	}
5453
5454
5455	/**
5456	* Raises a FPU stack overflow exception on a push.
5457	*
5458	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5459	* @param uFpuOpcode The FPU opcode value.
5460	*/
5461	void iemFpuStackPushOverflow(PVMCPUCC pVCpu, uint16_t uFpuOpcode) RT_NOEXCEPT
5462	{
5463	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5464	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5465	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5466	}
5467
5468
5469	/**
5470	* Raises a FPU stack overflow exception on a push with a memory operand.
5471	*
5472	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5473	* @param iEffSeg The effective memory operand selector register.
5474	* @param GCPtrEff The effective memory operand offset.
5475	* @param uFpuOpcode The FPU opcode value.
5476	*/
5477	void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff, uint16_t uFpuOpcode) RT_NOEXCEPT
5478	{
5479	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5480	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5481	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, uFpuOpcode);
5482	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5483	}
5484
5485	/** @} */
5486
5487
5488	/** @name SSE+AVX SIMD access and helpers.
5489	*
5490	* @{
5491	*/
5492	/**
5493	* Stores a result in a SIMD XMM register, updates the MXCSR.
5494	*
5495	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5496	* @param pResult The result to store.
5497	* @param iXmmReg Which SIMD XMM register to store the result in.
5498	*/
5499	void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT
5500	{
5501	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5502	pFpuCtx->MXCSR \|= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS;
5503
5504	/* The result is only updated if there is no unmasked exception pending. */
5505	if (( ~((pFpuCtx->MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT)
5506	& (pFpuCtx->MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)
5507	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult;
5508	}
5509
5510
5511	/**
5512	* Updates the MXCSR.
5513	*
5514	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5515	* @param fMxcsr The new MXCSR value.
5516	*/
5517	void iemSseUpdateMxcsr(PVMCPUCC pVCpu, uint32_t fMxcsr) RT_NOEXCEPT
5518	{
5519	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5520	pFpuCtx->MXCSR \|= fMxcsr & X86_MXCSR_XCPT_FLAGS;
5521	}
5522	/** @} */
5523
5524
5525	/** @name Memory access.
5526	*
5527	* @{
5528	*/
5529
5530	#undef LOG_GROUP
5531	#define LOG_GROUP LOG_GROUP_IEM_MEM
5532
5533	/**
5534	* Updates the IEMCPU::cbWritten counter if applicable.
5535	*
5536	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5537	* @param fAccess The access being accounted for.
5538	* @param cbMem The access size.
5539	*/
5540	DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
5541	{
5542	if ( (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK \| IEM_ACCESS_TYPE_WRITE)
5543	\|\| (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_TYPE_WRITE) )
5544	pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
5545	}
5546
5547
5548	/**
5549	* Applies the segment limit, base and attributes.
5550	*
5551	* This may raise a \#GP or \#SS.
5552	*
5553	* @returns VBox strict status code.
5554	*
5555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5556	* @param fAccess The kind of access which is being performed.
5557	* @param iSegReg The index of the segment register to apply.
5558	* This is UINT8_MAX if none (for IDT, GDT, LDT,
5559	* TSS, ++).
5560	* @param cbMem The access size.
5561	* @param pGCPtrMem Pointer to the guest memory address to apply
5562	* segmentation to. Input and output parameter.
5563	*/
5564	VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT
5565	{
5566	if (iSegReg == UINT8_MAX)
5567	return VINF_SUCCESS;
5568
5569	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
5570	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
5571	switch (IEM_GET_CPU_MODE(pVCpu))
5572	{
5573	case IEMMODE_16BIT:
5574	case IEMMODE_32BIT:
5575	{
5576	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
5577	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
5578
5579	if ( pSel->Attr.n.u1Present
5580	&& !pSel->Attr.n.u1Unusable)
5581	{
5582	Assert(pSel->Attr.n.u1DescType);
5583	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5584	{
5585	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5586	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
5587	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5588
5589	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5590	{
5591	/** @todo CPL check. */
5592	}
5593
5594	/*
5595	* There are two kinds of data selectors, normal and expand down.
5596	*/
5597	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5598	{
5599	if ( GCPtrFirst32 > pSel->u32Limit
5600	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5601	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5602	}
5603	else
5604	{
5605	/*
5606	* The upper boundary is defined by the B bit, not the G bit!
5607	*/
5608	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5609	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5610	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5611	}
5612	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5613	}
5614	else
5615	{
5616	/*
5617	* Code selector and usually be used to read thru, writing is
5618	* only permitted in real and V8086 mode.
5619	*/
5620	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5621	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
5622	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
5623	&& !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
5624	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5625
5626	if ( GCPtrFirst32 > pSel->u32Limit
5627	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5628	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5629
5630	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5631	{
5632	/** @todo CPL check. */
5633	}
5634
5635	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5636	}
5637	}
5638	else
5639	return iemRaiseGeneralProtectionFault0(pVCpu);
5640	return VINF_SUCCESS;
5641	}
5642
5643	case IEMMODE_64BIT:
5644	{
5645	RTGCPTR GCPtrMem = *pGCPtrMem;
5646	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
5647	*pGCPtrMem = GCPtrMem + pSel->u64Base;
5648
5649	Assert(cbMem >= 1);
5650	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
5651	return VINF_SUCCESS;
5652	/** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
5653	* 4.12.2 "Data Limit Checks in 64-bit Mode". */
5654	return iemRaiseGeneralProtectionFault0(pVCpu);
5655	}
5656
5657	default:
5658	AssertFailedReturn(VERR_IEM_IPE_7);
5659	}
5660	}
5661
5662
5663	/**
5664	* Translates a virtual address to a physical physical address and checks if we
5665	* can access the page as specified.
5666	*
5667	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5668	* @param GCPtrMem The virtual address.
5669	* @param cbAccess The access size, for raising \#PF correctly for
5670	* FXSAVE and such.
5671	* @param fAccess The intended access.
5672	* @param pGCPhysMem Where to return the physical address.
5673	*/
5674	VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t cbAccess,
5675	uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT
5676	{
5677	/** @todo Need a different PGM interface here. We're currently using
5678	* generic / REM interfaces. this won't cut it for R0. */
5679	/** @todo If/when PGM handles paged real-mode, we can remove the hack in
5680	* iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
5681	* here. */
5682	PGMPTWALK Walk;
5683	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5684	if (RT_FAILURE(rc))
5685	{
5686	LogEx(LOG_GROUP_IEM,("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
5687	/** @todo Check unassigned memory in unpaged mode. */
5688	/** @todo Reserved bits in page tables. Requires new PGM interface. */
5689	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5690	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5691	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
5692	#endif
5693	*pGCPhysMem = NIL_RTGCPHYS;
5694	return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess, rc);
5695	}
5696
5697	/* If the page is writable and does not have the no-exec bit set, all
5698	access is allowed. Otherwise we'll have to check more carefully... */
5699	if ((Walk.fEffective & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
5700	{
5701	/* Write to read only memory? */
5702	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5703	&& !(Walk.fEffective & X86_PTE_RW)
5704	&& ( ( IEM_GET_CPL(pVCpu) == 3
5705	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5706	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
5707	{
5708	LogEx(LOG_GROUP_IEM,("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
5709	*pGCPhysMem = NIL_RTGCPHYS;
5710	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5711	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5712	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5713	#endif
5714	return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
5715	}
5716
5717	/* Kernel memory accessed by userland? */
5718	if ( !(Walk.fEffective & X86_PTE_US)
5719	&& IEM_GET_CPL(pVCpu) == 3
5720	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5721	{
5722	LogEx(LOG_GROUP_IEM,("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
5723	*pGCPhysMem = NIL_RTGCPHYS;
5724	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5725	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5726	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5727	#endif
5728	return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess, VERR_ACCESS_DENIED);
5729	}
5730
5731	/* Executing non-executable memory? */
5732	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
5733	&& (Walk.fEffective & X86_PTE_PAE_NX)
5734	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
5735	{
5736	LogEx(LOG_GROUP_IEM,("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
5737	*pGCPhysMem = NIL_RTGCPHYS;
5738	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5739	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5740	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5741	#endif
5742	return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
5743	VERR_ACCESS_DENIED);
5744	}
5745	}
5746
5747	/*
5748	* Set the dirty / access flags.
5749	* ASSUMES this is set when the address is translated rather than on committ...
5750	*/
5751	/** @todo testcase: check when A and D bits are actually set by the CPU. */
5752	uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
5753	if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
5754	{
5755	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
5756	AssertRC(rc2);
5757	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
5758	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
5759	}
5760
5761	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
5762	*pGCPhysMem = GCPhys;
5763	return VINF_SUCCESS;
5764	}
5765
5766	#if 0 /unused/
5767	/**
5768	* Looks up a memory mapping entry.
5769	*
5770	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
5771	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5772	* @param pvMem The memory address.
5773	* @param fAccess The access to.
5774	*/
5775	DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
5776	{
5777	Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5778	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
5779	if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
5780	&& (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5781	return 0;
5782	if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
5783	&& (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5784	return 1;
5785	if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
5786	&& (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5787	return 2;
5788	return VERR_NOT_FOUND;
5789	}
5790	#endif
5791
5792	/**
5793	* Finds a free memmap entry when using iNextMapping doesn't work.
5794	*
5795	* @returns Memory mapping index, 1024 on failure.
5796	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5797	*/
5798	static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
5799	{
5800	/*
5801	* The easy case.
5802	*/
5803	if (pVCpu->iem.s.cActiveMappings == 0)
5804	{
5805	pVCpu->iem.s.iNextMapping = 1;
5806	return 0;
5807	}
5808
5809	/* There should be enough mappings for all instructions. */
5810	AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
5811
5812	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
5813	if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
5814	return i;
5815
5816	AssertFailedReturn(1024);
5817	}
5818
5819
5820	/**
5821	* Commits a bounce buffer that needs writing back and unmaps it.
5822	*
5823	* @returns Strict VBox status code.
5824	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5825	* @param iMemMap The index of the buffer to commit.
5826	* @param fPostponeFail Whether we can postpone writer failures to ring-3.
5827	* Always false in ring-3, obviously.
5828	*/
5829	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
5830	{
5831	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
5832	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
5833	#ifdef IN_RING3
5834	Assert(!fPostponeFail);
5835	RT_NOREF_PV(fPostponeFail);
5836	#endif
5837
5838	/*
5839	* Do the writing.
5840	*/
5841	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5842	if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
5843	{
5844	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
5845	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5846	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5847	if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
5848	{
5849	/*
5850	* Carefully and efficiently dealing with access handler return
5851	* codes make this a little bloated.
5852	*/
5853	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
5854	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5855	pbBuf,
5856	cbFirst,
5857	PGMACCESSORIGIN_IEM);
5858	if (rcStrict == VINF_SUCCESS)
5859	{
5860	if (cbSecond)
5861	{
5862	rcStrict = PGMPhysWrite(pVM,
5863	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5864	pbBuf + cbFirst,
5865	cbSecond,
5866	PGMACCESSORIGIN_IEM);
5867	if (rcStrict == VINF_SUCCESS)
5868	{ /* nothing */ }
5869	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5870	{
5871	LogEx(LOG_GROUP_IEM,
5872	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
5873	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5874	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5875	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5876	}
5877	#ifndef IN_RING3
5878	else if (fPostponeFail)
5879	{
5880	LogEx(LOG_GROUP_IEM,
5881	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5882	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5883	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5884	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5885	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5886	return iemSetPassUpStatus(pVCpu, rcStrict);
5887	}
5888	#endif
5889	else
5890	{
5891	LogEx(LOG_GROUP_IEM,
5892	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5893	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5894	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5895	return rcStrict;
5896	}
5897	}
5898	}
5899	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5900	{
5901	if (!cbSecond)
5902	{
5903	LogEx(LOG_GROUP_IEM,
5904	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
5905	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5906	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5907	}
5908	else
5909	{
5910	VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
5911	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5912	pbBuf + cbFirst,
5913	cbSecond,
5914	PGMACCESSORIGIN_IEM);
5915	if (rcStrict2 == VINF_SUCCESS)
5916	{
5917	LogEx(LOG_GROUP_IEM,
5918	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
5919	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5920	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5921	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5922	}
5923	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5924	{
5925	LogEx(LOG_GROUP_IEM,
5926	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
5927	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5928	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5929	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5930	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5931	}
5932	#ifndef IN_RING3
5933	else if (fPostponeFail)
5934	{
5935	LogEx(LOG_GROUP_IEM,
5936	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5937	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5938	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5939	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5940	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5941	return iemSetPassUpStatus(pVCpu, rcStrict);
5942	}
5943	#endif
5944	else
5945	{
5946	LogEx(LOG_GROUP_IEM,
5947	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5948	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5949	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5950	return rcStrict2;
5951	}
5952	}
5953	}
5954	#ifndef IN_RING3
5955	else if (fPostponeFail)
5956	{
5957	LogEx(LOG_GROUP_IEM,
5958	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5959	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5960	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5961	if (!cbSecond)
5962	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST;
5963	else
5964	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND;
5965	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5966	return iemSetPassUpStatus(pVCpu, rcStrict);
5967	}
5968	#endif
5969	else
5970	{
5971	LogEx(LOG_GROUP_IEM,
5972	("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5973	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5974	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5975	return rcStrict;
5976	}
5977	}
5978	else
5979	{
5980	/*
5981	* No access handlers, much simpler.
5982	*/
5983	int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
5984	if (RT_SUCCESS(rc))
5985	{
5986	if (cbSecond)
5987	{
5988	rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
5989	if (RT_SUCCESS(rc))
5990	{ /* likely */ }
5991	else
5992	{
5993	LogEx(LOG_GROUP_IEM,
5994	("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5995	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5996	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
5997	return rc;
5998	}
5999	}
6000	}
6001	else
6002	{
6003	LogEx(LOG_GROUP_IEM,
6004	("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
6005	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
6006	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
6007	return rc;
6008	}
6009	}
6010	}
6011
6012	#if defined(IEM_LOG_MEMORY_WRITES)
6013	Log5(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
6014	RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
6015	if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
6016	Log5(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
6017	RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
6018	&pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
6019
6020	size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
6021	g_cbIemWrote = cbWrote;
6022	memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
6023	#endif
6024
6025	/*
6026	* Free the mapping entry.
6027	*/
6028	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6029	Assert(pVCpu->iem.s.cActiveMappings != 0);
6030	pVCpu->iem.s.cActiveMappings--;
6031	return VINF_SUCCESS;
6032	}
6033
6034
6035	/**
6036	* iemMemMap worker that deals with a request crossing pages.
6037	*/
6038	static VBOXSTRICTRC
6039	iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void *ppvMem, uint8_t pbUnmapInfo,
6040	size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
6041	{
6042	Assert(cbMem <= GUEST_PAGE_SIZE);
6043
6044	/*
6045	* Do the address translations.
6046	*/
6047	uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (uint32_t)(GCPtrFirst & GUEST_PAGE_OFFSET_MASK);
6048	RTGCPHYS GCPhysFirst;
6049	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, cbFirstPage, fAccess, &GCPhysFirst);
6050	if (rcStrict != VINF_SUCCESS)
6051	return rcStrict;
6052	Assert((GCPhysFirst & GUEST_PAGE_OFFSET_MASK) == (GCPtrFirst & GUEST_PAGE_OFFSET_MASK));
6053
6054	uint32_t const cbSecondPage = (uint32_t)cbMem - cbFirstPage;
6055	RTGCPHYS GCPhysSecond;
6056	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
6057	cbSecondPage, fAccess, &GCPhysSecond);
6058	if (rcStrict != VINF_SUCCESS)
6059	return rcStrict;
6060	Assert((GCPhysSecond & GUEST_PAGE_OFFSET_MASK) == 0);
6061	GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; /** @todo why? */
6062
6063	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6064
6065	/*
6066	* Read in the current memory content if it's a read, execute or partial
6067	* write access.
6068	*/
6069	uint8_t * const pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
6070
6071	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
6072	{
6073	if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
6074	{
6075	/*
6076	* Must carefully deal with access handler status codes here,
6077	* makes the code a bit bloated.
6078	*/
6079	rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
6080	if (rcStrict == VINF_SUCCESS)
6081	{
6082	rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
6083	if (rcStrict == VINF_SUCCESS)
6084	{ /likely / }
6085	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
6086	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
6087	else
6088	{
6089	LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
6090	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
6091	return rcStrict;
6092	}
6093	}
6094	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
6095	{
6096	VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
6097	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
6098	{
6099	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
6100	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
6101	}
6102	else
6103	{
6104	LogEx(LOG_GROUP_IEM,
6105	("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
6106	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
6107	return rcStrict2;
6108	}
6109	}
6110	else
6111	{
6112	LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
6113	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
6114	return rcStrict;
6115	}
6116	}
6117	else
6118	{
6119	/*
6120	* No informational status codes here, much more straight forward.
6121	*/
6122	int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
6123	if (RT_SUCCESS(rc))
6124	{
6125	Assert(rc == VINF_SUCCESS);
6126	rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
6127	if (RT_SUCCESS(rc))
6128	Assert(rc == VINF_SUCCESS);
6129	else
6130	{
6131	LogEx(LOG_GROUP_IEM,
6132	("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
6133	return rc;
6134	}
6135	}
6136	else
6137	{
6138	LogEx(LOG_GROUP_IEM,
6139	("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
6140	return rc;
6141	}
6142	}
6143	}
6144	#ifdef VBOX_STRICT
6145	else
6146	memset(pbBuf, 0xcc, cbMem);
6147	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
6148	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
6149	#endif
6150	AssertCompileMemberAlignment(VMCPU, iem.s.aBounceBuffers, 64);
6151
6152	/*
6153	* Commit the bounce buffer entry.
6154	*/
6155	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
6156	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
6157	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
6158	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
6159	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
6160	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
6161	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
6162	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6163	pVCpu->iem.s.cActiveMappings++;
6164
6165	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6166	*ppvMem = pbBuf;
6167	*pbUnmapInfo = iMemMap \| 0x08 \| ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);
6168	return VINF_SUCCESS;
6169	}
6170
6171
6172	/**
6173	* iemMemMap woker that deals with iemMemPageMap failures.
6174	*/
6175	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void *ppvMem, uint8_t pbUnmapInfo, size_t cbMem,
6176	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
6177	{
6178	/*
6179	* Filter out conditions we can handle and the ones which shouldn't happen.
6180	*/
6181	if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
6182	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
6183	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
6184	{
6185	AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
6186	return rcMap;
6187	}
6188	pVCpu->iem.s.cPotentialExits++;
6189
6190	/*
6191	* Read in the current memory content if it's a read, execute or partial
6192	* write access.
6193	*/
6194	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
6195	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
6196	{
6197	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
6198	memset(pbBuf, 0xff, cbMem);
6199	else
6200	{
6201	int rc;
6202	if (!(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS))
6203	{
6204	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
6205	if (rcStrict == VINF_SUCCESS)
6206	{ /* nothing */ }
6207	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
6208	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
6209	else
6210	{
6211	LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
6212	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
6213	return rcStrict;
6214	}
6215	}
6216	else
6217	{
6218	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
6219	if (RT_SUCCESS(rc))
6220	{ /* likely */ }
6221	else
6222	{
6223	LogEx(LOG_GROUP_IEM, ("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
6224	GCPhysFirst, rc));
6225	return rc;
6226	}
6227	}
6228	}
6229	}
6230	#ifdef VBOX_STRICT
6231	else
6232	memset(pbBuf, 0xcc, cbMem);
6233	#endif
6234	#ifdef VBOX_STRICT
6235	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
6236	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
6237	#endif
6238
6239	/*
6240	* Commit the bounce buffer entry.
6241	*/
6242	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
6243	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
6244	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
6245	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
6246	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
6247	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
6248	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
6249	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6250	pVCpu->iem.s.cActiveMappings++;
6251
6252	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6253	*ppvMem = pbBuf;
6254	*pbUnmapInfo = iMemMap \| 0x08 \| ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);
6255	return VINF_SUCCESS;
6256	}
6257
6258
6259
6260	/**
6261	* Maps the specified guest memory for the given kind of access.
6262	*
6263	* This may be using bounce buffering of the memory if it's crossing a page
6264	* boundary or if there is an access handler installed for any of it. Because
6265	* of lock prefix guarantees, we're in for some extra clutter when this
6266	* happens.
6267	*
6268	* This may raise a \#GP, \#SS, \#PF or \#AC.
6269	*
6270	* @returns VBox strict status code.
6271	*
6272	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6273	* @param ppvMem Where to return the pointer to the mapped memory.
6274	* @param pbUnmapInfo Where to return unmap info to be passed to
6275	* iemMemCommitAndUnmap or iemMemRollbackAndUnmap when
6276	* done.
6277	* @param cbMem The number of bytes to map. This is usually 1, 2, 4, 6,
6278	* 8, 12, 16, 32 or 512. When used by string operations
6279	* it can be up to a page.
6280	* @param iSegReg The index of the segment register to use for this
6281	* access. The base and limits are checked. Use UINT8_MAX
6282	* to indicate that no segmentation is required (for IDT,
6283	* GDT and LDT accesses).
6284	* @param GCPtrMem The address of the guest memory.
6285	* @param fAccess How the memory is being accessed. The
6286	* IEM_ACCESS_TYPE_XXX part is used to figure out how to
6287	* map the memory, while the IEM_ACCESS_WHAT_XXX part is
6288	* used when raising exceptions. The IEM_ACCESS_ATOMIC and
6289	* IEM_ACCESS_PARTIAL_WRITE bits are also allowed to be
6290	* set.
6291	* @param uAlignCtl Alignment control:
6292	* - Bits 15:0 is the alignment mask.
6293	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
6294	* IEM_MEMMAP_F_ALIGN_SSE, and
6295	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
6296	* Pass zero to skip alignment.
6297	*/
6298	VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void *ppvMem, uint8_t pbUnmapInfo, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
6299	uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT
6300	{
6301	/*
6302	* Check the input and figure out which mapping entry to use.
6303	*/
6304	Assert(cbMem <= sizeof(pVCpu->iem.s.aBounceBuffers[0]));
6305	Assert( cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 256 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 102 \|\| cbMem == 94
6306	\|\| (iSegReg == UINT8_MAX && uAlignCtl == 0 && fAccess == IEM_ACCESS_DATA_R /* for the CPUID logging interface */) );
6307	Assert(!(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_ATOMIC \| IEM_ACCESS_PARTIAL_WRITE)));
6308	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
6309
6310	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
6311	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6312	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6313	{
6314	iMemMap = iemMemMapFindFree(pVCpu);
6315	AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
6316	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
6317	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
6318	pVCpu->iem.s.aMemMappings[2].fAccess),
6319	VERR_IEM_IPE_9);
6320	}
6321
6322	/*
6323	* Map the memory, checking that we can actually access it. If something
6324	* slightly complicated happens, fall back on bounce buffering.
6325	*/
6326	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6327	if (rcStrict == VINF_SUCCESS)
6328	{ /* likely */ }
6329	else
6330	return rcStrict;
6331
6332	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */
6333	{ /* likely */ }
6334	else
6335	return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, pbUnmapInfo, cbMem, GCPtrMem, fAccess);
6336
6337	/*
6338	* Alignment check.
6339	*/
6340	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
6341	{ /* likelyish */ }
6342	else
6343	{
6344	/* Misaligned access. */
6345	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6346	{
6347	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
6348	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
6349	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
6350	{
6351	AssertCompile(X86_CR0_AM == X86_EFL_AC);
6352
6353	if (iemMemAreAlignmentChecksEnabled(pVCpu))
6354	return iemRaiseAlignmentCheckException(pVCpu);
6355	}
6356	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
6357	&& (GCPtrMem & 3) /* The value 4 matches 10980xe's FXSAVE and helps make bs3-cpu-basic2 work. */
6358	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
6359	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. Using 4 for now as
6360	* that's what FXSAVE does on a 10980xe. */
6361	&& iemMemAreAlignmentChecksEnabled(pVCpu))
6362	return iemRaiseAlignmentCheckException(pVCpu);
6363	else
6364	return iemRaiseGeneralProtectionFault0(pVCpu);
6365	}
6366
6367	#if (defined(RT_ARCH_AMD64) && defined(RT_OS_LINUX)) \|\| defined(RT_ARCH_ARM64)
6368	/* If the access is atomic there are host platform alignmnet restrictions
6369	we need to conform with. */
6370	if ( !(fAccess & IEM_ACCESS_ATOMIC)
6371	# if defined(RT_ARCH_AMD64)
6372	\|\| (64U - (GCPtrMem & 63U) >= cbMem) /* split-lock detection. ASSUMES 64 byte cache line. */
6373	# elif defined(RT_ARCH_ARM64)
6374	\|\| (16U - (GCPtrMem & 15U) >= cbMem) /* LSE2 allows atomics anywhere within a 16 byte sized & aligned block. */
6375	# else
6376	# error port me
6377	# endif
6378	)
6379	{ /* okay */ }
6380	else
6381	{
6382	LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv LB %u - misaligned atomic fallback.\n", GCPtrMem, cbMem));
6383	pVCpu->iem.s.cMisalignedAtomics += 1;
6384	return VINF_EM_EMULATE_SPLIT_LOCK;
6385	}
6386	#endif
6387	}
6388
6389	#ifdef IEM_WITH_DATA_TLB
6390	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
6391
6392	/*
6393	* Get the TLB entry for this page.
6394	*/
6395	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6396	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6397	if (pTlbe->uTag == uTag)
6398	{
6399	# ifdef VBOX_WITH_STATISTICS
6400	pVCpu->iem.s.DataTlb.cTlbHits++;
6401	# endif
6402	}
6403	else
6404	{
6405	pVCpu->iem.s.DataTlb.cTlbMisses++;
6406	PGMPTWALK Walk;
6407	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6408	if (RT_FAILURE(rc))
6409	{
6410	LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6411	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6412	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6413	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6414	# endif
6415	return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, rc);
6416	}
6417
6418	Assert(Walk.fSucceeded);
6419	pTlbe->uTag = uTag;
6420	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6421	pTlbe->GCPhys = Walk.GCPhys;
6422	pTlbe->pbMappingR3 = NULL;
6423	}
6424
6425	/*
6426	* Check TLB page table level access flags.
6427	*/
6428	/* If the page is either supervisor only or non-writable, we need to do
6429	more careful access checks. */
6430	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_WRITE))
6431	{
6432	/* Write to read only memory? */
6433	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
6434	&& (fAccess & IEM_ACCESS_TYPE_WRITE)
6435	&& ( ( IEM_GET_CPL(pVCpu) == 3
6436	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6437	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
6438	{
6439	LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6440	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6441	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6442	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6443	# endif
6444	return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6445	}
6446
6447	/* Kernel memory accessed by userland? */
6448	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
6449	&& IEM_GET_CPL(pVCpu) == 3
6450	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6451	{
6452	LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6453	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6454	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6455	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6456	# endif
6457	return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, VERR_ACCESS_DENIED);
6458	}
6459	}
6460
6461	/*
6462	* Set the dirty / access flags.
6463	* ASSUMES this is set when the address is translated rather than on commit...
6464	*/
6465	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6466	uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) \| IEMTLBE_F_PT_NO_ACCESSED;
6467	if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty)
6468	{
6469	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6470	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6471	AssertRC(rc2);
6472	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6473	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6474	pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty;
6475	}
6476
6477	/*
6478	* Look up the physical page info if necessary.
6479	*/
6480	uint8_t *pbMem = NULL;
6481	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6482	# ifdef IN_RING3
6483	pbMem = pTlbe->pbMappingR3;
6484	# else
6485	pbMem = NULL;
6486	# endif
6487	else
6488	{
6489	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6490	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6491	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6492	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6493	AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
6494	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
6495	{ /* likely */ }
6496	else
6497	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
6498	pTlbe->pbMappingR3 = NULL;
6499	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6500	\| IEMTLBE_F_NO_MAPPINGR3
6501	\| IEMTLBE_F_PG_NO_READ
6502	\| IEMTLBE_F_PG_NO_WRITE
6503	\| IEMTLBE_F_PG_UNASSIGNED
6504	\| IEMTLBE_F_PG_CODE_PAGE);
6505	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6506	&pbMem, &pTlbe->fFlagsAndPhysRev);
6507	AssertRCReturn(rc, rc);
6508	# ifdef IN_RING3
6509	pTlbe->pbMappingR3 = pbMem;
6510	# endif
6511	}
6512
6513	/*
6514	* Check the physical page level access and mapping.
6515	*/
6516	if ( !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))
6517	\|\| !(pTlbe->fFlagsAndPhysRev & ( (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0)
6518	\| (fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0))) )
6519	{ /* probably likely */ }
6520	else
6521	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, pbUnmapInfo, cbMem,
6522	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6523	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6524	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6525	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6526	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6527
6528	if (pbMem)
6529	{
6530	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6531	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6532	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6533	}
6534	else
6535	{
6536	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6537	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6538	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6539	if (rcStrict != VINF_SUCCESS)
6540	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, pbUnmapInfo, cbMem, GCPhysFirst, fAccess, rcStrict);
6541	}
6542
6543	void * const pvMem = pbMem;
6544
6545	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6546	Log6(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6547	if (fAccess & IEM_ACCESS_TYPE_READ)
6548	Log2(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6549
6550	#else /* !IEM_WITH_DATA_TLB */
6551
6552	RTGCPHYS GCPhysFirst;
6553	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, &GCPhysFirst);
6554	if (rcStrict != VINF_SUCCESS)
6555	return rcStrict;
6556
6557	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6558	Log6(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6559	if (fAccess & IEM_ACCESS_TYPE_READ)
6560	Log2(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6561
6562	void *pvMem;
6563	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6564	if (rcStrict != VINF_SUCCESS)
6565	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, pbUnmapInfo, cbMem, GCPhysFirst, fAccess, rcStrict);
6566
6567	#endif /* !IEM_WITH_DATA_TLB */
6568
6569	/*
6570	* Fill in the mapping table entry.
6571	*/
6572	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6573	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6574	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6575	pVCpu->iem.s.cActiveMappings += 1;
6576
6577	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6578	*ppvMem = pvMem;
6579	*pbUnmapInfo = iMemMap \| 0x08 \| ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);
6580	AssertCompile(IEM_ACCESS_TYPE_MASK <= 0xf);
6581	AssertCompile(RT_ELEMENTS(pVCpu->iem.s.aMemMappings) < 8);
6582
6583	return VINF_SUCCESS;
6584	}
6585
6586
6587	/**
6588	* Commits the guest memory if bounce buffered and unmaps it.
6589	*
6590	* @returns Strict VBox status code.
6591	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6592	* @param bUnmapInfo Unmap info set by iemMemMap.
6593	*/
6594	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT
6595	{
6596	uintptr_t const iMemMap = bUnmapInfo & 0x7;
6597	AssertMsgReturn( (bUnmapInfo & 0x08)
6598	&& iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6599	&& (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK \| 0xf)) == ((unsigned)bUnmapInfo >> 4),
6600	("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess),
6601	VERR_NOT_FOUND);
6602
6603	/* If it's bounce buffered, we may need to write back the buffer. */
6604	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6605	{
6606	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6607	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6608	}
6609	/* Otherwise unlock it. */
6610	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6611	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6612
6613	/* Free the entry. */
6614	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6615	Assert(pVCpu->iem.s.cActiveMappings != 0);
6616	pVCpu->iem.s.cActiveMappings--;
6617	return VINF_SUCCESS;
6618	}
6619
6620
6621	/**
6622	* Rolls back the guest memory (conceptually only) and unmaps it.
6623	*
6624	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6625	* @param bUnmapInfo Unmap info set by iemMemMap.
6626	*/
6627	void iemMemRollbackAndUnmap(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT
6628	{
6629	uintptr_t const iMemMap = bUnmapInfo & 0x7;
6630	AssertMsgReturnVoid( (bUnmapInfo & 0x08)
6631	&& iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6632	&& (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK \| 0xf))
6633	== ((unsigned)bUnmapInfo >> 4),
6634	("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess));
6635
6636	/* Unlock it if necessary. */
6637	if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6638	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6639
6640	/* Free the entry. */
6641	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6642	Assert(pVCpu->iem.s.cActiveMappings != 0);
6643	pVCpu->iem.s.cActiveMappings--;
6644	}
6645
6646	#ifdef IEM_WITH_SETJMP
6647
6648	/**
6649	* Maps the specified guest memory for the given kind of access, longjmp on
6650	* error.
6651	*
6652	* This may be using bounce buffering of the memory if it's crossing a page
6653	* boundary or if there is an access handler installed for any of it. Because
6654	* of lock prefix guarantees, we're in for some extra clutter when this
6655	* happens.
6656	*
6657	* This may raise a \#GP, \#SS, \#PF or \#AC.
6658	*
6659	* @returns Pointer to the mapped memory.
6660	*
6661	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6662	* @param bUnmapInfo Where to return unmap info to be passed to
6663	* iemMemCommitAndUnmapJmp, iemMemCommitAndUnmapRwSafeJmp,
6664	* iemMemCommitAndUnmapWoSafeJmp,
6665	* iemMemCommitAndUnmapRoSafeJmp,
6666	* iemMemRollbackAndUnmapWoSafe or iemMemRollbackAndUnmap
6667	* when done.
6668	* @param cbMem The number of bytes to map. This is usually 1,
6669	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
6670	* string operations it can be up to a page.
6671	* @param iSegReg The index of the segment register to use for
6672	* this access. The base and limits are checked.
6673	* Use UINT8_MAX to indicate that no segmentation
6674	* is required (for IDT, GDT and LDT accesses).
6675	* @param GCPtrMem The address of the guest memory.
6676	* @param fAccess How the memory is being accessed. The
6677	* IEM_ACCESS_TYPE_XXX part is used to figure out how to
6678	* map the memory, while the IEM_ACCESS_WHAT_XXX part is
6679	* used when raising exceptions. The IEM_ACCESS_ATOMIC and
6680	* IEM_ACCESS_PARTIAL_WRITE bits are also allowed to be
6681	* set.
6682	* @param uAlignCtl Alignment control:
6683	* - Bits 15:0 is the alignment mask.
6684	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
6685	* IEM_MEMMAP_F_ALIGN_SSE, and
6686	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
6687	* Pass zero to skip alignment.
6688	*/
6689	void iemMemMapJmp(PVMCPUCC pVCpu, uint8_t pbUnmapInfo, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess,
6690	uint32_t uAlignCtl) IEM_NOEXCEPT_MAY_LONGJMP
6691	{
6692	/*
6693	* Check the input, check segment access and adjust address
6694	* with segment base.
6695	*/
6696	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 94); /* 512 is the max! */
6697	Assert(!(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_ATOMIC \| IEM_ACCESS_PARTIAL_WRITE)));
6698	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
6699
6700	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6701	if (rcStrict == VINF_SUCCESS) { /likely/ }
6702	else IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
6703
6704	/*
6705	* Alignment check.
6706	*/
6707	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
6708	{ /* likelyish */ }
6709	else
6710	{
6711	/* Misaligned access. */
6712	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6713	{
6714	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
6715	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
6716	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
6717	{
6718	AssertCompile(X86_CR0_AM == X86_EFL_AC);
6719
6720	if (iemMemAreAlignmentChecksEnabled(pVCpu))
6721	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6722	}
6723	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
6724	&& (GCPtrMem & 3) /* The value 4 matches 10980xe's FXSAVE and helps make bs3-cpu-basic2 work. */
6725	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
6726	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. Using 4 for now as
6727	* that's what FXSAVE does on a 10980xe. */
6728	&& iemMemAreAlignmentChecksEnabled(pVCpu))
6729	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6730	else
6731	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
6732	}
6733
6734	#if (defined(RT_ARCH_AMD64) && defined(RT_OS_LINUX)) \|\| defined(RT_ARCH_ARM64)
6735	/* If the access is atomic there are host platform alignmnet restrictions
6736	we need to conform with. */
6737	if ( !(fAccess & IEM_ACCESS_ATOMIC)
6738	# if defined(RT_ARCH_AMD64)
6739	\|\| (64U - (GCPtrMem & 63U) >= cbMem) /* split-lock detection. ASSUMES 64 byte cache line. */
6740	# elif defined(RT_ARCH_ARM64)
6741	\|\| (16U - (GCPtrMem & 15U) >= cbMem) /* LSE2 allows atomics anywhere within a 16 byte sized & aligned block. */
6742	# else
6743	# error port me
6744	# endif
6745	)
6746	{ /* okay */ }
6747	else
6748	{
6749	LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv LB %u - misaligned atomic fallback.\n", GCPtrMem, cbMem));
6750	pVCpu->iem.s.cMisalignedAtomics += 1;
6751	IEM_DO_LONGJMP(pVCpu, VINF_EM_EMULATE_SPLIT_LOCK);
6752	}
6753	#endif
6754	}
6755
6756	/*
6757	* Figure out which mapping entry to use.
6758	*/
6759	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
6760	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6761	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6762	{
6763	iMemMap = iemMemMapFindFree(pVCpu);
6764	AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
6765	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
6766	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
6767	pVCpu->iem.s.aMemMappings[2].fAccess),
6768	IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_9));
6769	}
6770
6771	/*
6772	* Crossing a page boundary?
6773	*/
6774	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
6775	{ /* No (likely). */ }
6776	else
6777	{
6778	void *pvMem;
6779	rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, pbUnmapInfo, cbMem, GCPtrMem, fAccess);
6780	if (rcStrict == VINF_SUCCESS)
6781	return pvMem;
6782	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
6783	}
6784
6785	#ifdef IEM_WITH_DATA_TLB
6786	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
6787
6788	/*
6789	* Get the TLB entry for this page.
6790	*/
6791	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6792	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6793	if (pTlbe->uTag == uTag)
6794	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6795	else
6796	{
6797	pVCpu->iem.s.DataTlb.cTlbMisses++;
6798	PGMPTWALK Walk;
6799	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6800	if (RT_FAILURE(rc))
6801	{
6802	LogEx(LOG_GROUP_IEM, ("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6803	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6804	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6805	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6806	# endif
6807	iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, rc);
6808	}
6809
6810	Assert(Walk.fSucceeded);
6811	pTlbe->uTag = uTag;
6812	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6813	pTlbe->GCPhys = Walk.GCPhys;
6814	pTlbe->pbMappingR3 = NULL;
6815	}
6816
6817	/*
6818	* Check the flags and physical revision.
6819	*/
6820	/** @todo make the caller pass these in with fAccess. */
6821	uint64_t const fNoUser = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && IEM_GET_CPL(pVCpu) == 3
6822	? IEMTLBE_F_PT_NO_USER : 0;
6823	uint64_t const fNoWriteNoDirty = fAccess & IEM_ACCESS_TYPE_WRITE
6824	? IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY
6825	\| ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)
6826	\|\| (IEM_GET_CPL(pVCpu) == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6827	? IEMTLBE_F_PT_NO_WRITE : 0)
6828	: 0;
6829	uint64_t const fNoRead = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0;
6830	uint8_t *pbMem = NULL;
6831	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PT_NO_ACCESSED \| fNoRead \| fNoWriteNoDirty \| fNoUser))
6832	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6833	# ifdef IN_RING3
6834	pbMem = pTlbe->pbMappingR3;
6835	# else
6836	pbMem = NULL;
6837	# endif
6838	else
6839	{
6840	/*
6841	* Okay, something isn't quite right or needs refreshing.
6842	*/
6843	/* Write to read only memory? */
6844	if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
6845	{
6846	LogEx(LOG_GROUP_IEM, ("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6847	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6848	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6849	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6850	# endif
6851	iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6852	}
6853
6854	/* Kernel memory accessed by userland? */
6855	if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER)
6856	{
6857	LogEx(LOG_GROUP_IEM, ("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6858	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6859	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6860	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6861	# endif
6862	iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, VERR_ACCESS_DENIED);
6863	}
6864
6865	/* Set the dirty / access flags.
6866	ASSUMES this is set when the address is translated rather than on commit... */
6867	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6868	if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED))
6869	{
6870	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6871	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6872	AssertRC(rc2);
6873	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6874	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6875	pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED);
6876	}
6877
6878	/*
6879	* Check if the physical page info needs updating.
6880	*/
6881	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6882	# ifdef IN_RING3
6883	pbMem = pTlbe->pbMappingR3;
6884	# else
6885	pbMem = NULL;
6886	# endif
6887	else
6888	{
6889	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6890	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6891	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6892	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6893	AssertCompile(PGMIEMGCPHYS2PTR_F_CODE_PAGE == IEMTLBE_F_PG_CODE_PAGE);
6894	pTlbe->pbMappingR3 = NULL;
6895	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6896	\| IEMTLBE_F_NO_MAPPINGR3
6897	\| IEMTLBE_F_PG_NO_READ
6898	\| IEMTLBE_F_PG_NO_WRITE
6899	\| IEMTLBE_F_PG_UNASSIGNED
6900	\| IEMTLBE_F_PG_CODE_PAGE);
6901	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6902	&pbMem, &pTlbe->fFlagsAndPhysRev);
6903	AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc));
6904	# ifdef IN_RING3
6905	pTlbe->pbMappingR3 = pbMem;
6906	# endif
6907	}
6908
6909	/*
6910	* Check the physical page level access and mapping.
6911	*/
6912	if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty \| fNoRead) & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))))
6913	{ /* probably likely */ }
6914	else
6915	{
6916	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, pbUnmapInfo, cbMem,
6917	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6918	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6919	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6920	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6921	if (rcStrict == VINF_SUCCESS)
6922	return pbMem;
6923	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
6924	}
6925	}
6926	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6927
6928	if (pbMem)
6929	{
6930	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6931	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6932	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6933	}
6934	else
6935	{
6936	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6937	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6938	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6939	if (rcStrict == VINF_SUCCESS)
6940	{
6941	*pbUnmapInfo = iMemMap \| 0x08 \| ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);
6942	return pbMem;
6943	}
6944	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
6945	}
6946
6947	void * const pvMem = pbMem;
6948
6949	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6950	Log6(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6951	if (fAccess & IEM_ACCESS_TYPE_READ)
6952	Log2(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6953
6954	#else /* !IEM_WITH_DATA_TLB */
6955
6956
6957	RTGCPHYS GCPhysFirst;
6958	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, &GCPhysFirst);
6959	if (rcStrict == VINF_SUCCESS) { /likely/ }
6960	else IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
6961
6962	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6963	Log6(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6964	if (fAccess & IEM_ACCESS_TYPE_READ)
6965	Log2(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6966
6967	void *pvMem;
6968	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6969	if (rcStrict == VINF_SUCCESS)
6970	{ /* likely */ }
6971	else
6972	{
6973	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, pbUnmapInfo, cbMem, GCPhysFirst, fAccess, rcStrict);
6974	if (rcStrict == VINF_SUCCESS)
6975	return pvMem;
6976	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
6977	}
6978
6979	#endif /* !IEM_WITH_DATA_TLB */
6980
6981	/*
6982	* Fill in the mapping table entry.
6983	*/
6984	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6985	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6986	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6987	pVCpu->iem.s.cActiveMappings++;
6988
6989	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6990
6991	*pbUnmapInfo = iMemMap \| 0x08 \| ((fAccess & IEM_ACCESS_TYPE_MASK) << 4);
6992	return pvMem;
6993	}
6994
6995
6996	/**
6997	* Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
6998	*
6999	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7000	* @param pvMem The mapping.
7001	* @param fAccess The kind of access.
7002	*/
7003	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
7004	{
7005	uintptr_t const iMemMap = bUnmapInfo & 0x7;
7006	AssertMsgReturnVoid( (bUnmapInfo & 0x08)
7007	&& iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
7008	&& (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK \| 0xf))
7009	== ((unsigned)bUnmapInfo >> 4),
7010	("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess));
7011
7012	/* If it's bounce buffered, we may need to write back the buffer. */
7013	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
7014	{
7015	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
7016	{
7017	VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
7018	if (rcStrict == VINF_SUCCESS)
7019	return;
7020	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
7021	}
7022	}
7023	/* Otherwise unlock it. */
7024	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
7025	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
7026
7027	/* Free the entry. */
7028	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
7029	Assert(pVCpu->iem.s.cActiveMappings != 0);
7030	pVCpu->iem.s.cActiveMappings--;
7031	}
7032
7033
7034	/** Fallback for iemMemCommitAndUnmapRwJmp. */
7035	void iemMemCommitAndUnmapRwSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
7036	{
7037	Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE));
7038	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7039	}
7040
7041
7042	/** Fallback for iemMemCommitAndUnmapAtJmp. */
7043	void iemMemCommitAndUnmapAtSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
7044	{
7045	Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE));
7046	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7047	}
7048
7049
7050	/** Fallback for iemMemCommitAndUnmapWoJmp. */
7051	void iemMemCommitAndUnmapWoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
7052	{
7053	Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_WRITE);
7054	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7055	}
7056
7057
7058	/** Fallback for iemMemCommitAndUnmapRoJmp. */
7059	void iemMemCommitAndUnmapRoSafeJmp(PVMCPUCC pVCpu, uint8_t bUnmapInfo) IEM_NOEXCEPT_MAY_LONGJMP
7060	{
7061	Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_READ);
7062	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7063	}
7064
7065
7066	/** Fallback for iemMemRollbackAndUnmapWo. */
7067	void iemMemRollbackAndUnmapWoSafe(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT
7068	{
7069	Assert(((bUnmapInfo >> 4) & IEM_ACCESS_TYPE_MASK) == IEM_ACCESS_TYPE_WRITE);
7070	iemMemRollbackAndUnmap(pVCpu, bUnmapInfo);
7071	}
7072
7073	#endif /* IEM_WITH_SETJMP */
7074
7075	#ifndef IN_RING3
7076	/**
7077	* Commits the guest memory if bounce buffered and unmaps it, if any bounce
7078	* buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
7079	*
7080	* Allows the instruction to be completed and retired, while the IEM user will
7081	* return to ring-3 immediately afterwards and do the postponed writes there.
7082	*
7083	* @returns VBox status code (no strict statuses). Caller must check
7084	* VMCPU_FF_IEM before repeating string instructions and similar stuff.
7085	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7086	* @param pvMem The mapping.
7087	* @param fAccess The kind of access.
7088	*/
7089	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT
7090	{
7091	uintptr_t const iMemMap = bUnmapInfo & 0x7;
7092	AssertMsgReturn( (bUnmapInfo & 0x08)
7093	&& iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
7094	&& (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_TYPE_MASK \| 0xf))
7095	== ((unsigned)bUnmapInfo >> 4),
7096	("%#x fAccess=%#x\n", bUnmapInfo, pVCpu->iem.s.aMemMappings[iMemMap].fAccess),
7097	VERR_NOT_FOUND);
7098
7099	/* If it's bounce buffered, we may need to write back the buffer. */
7100	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
7101	{
7102	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
7103	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /fPostponeFail/);
7104	}
7105	/* Otherwise unlock it. */
7106	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
7107	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
7108
7109	/* Free the entry. */
7110	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
7111	Assert(pVCpu->iem.s.cActiveMappings != 0);
7112	pVCpu->iem.s.cActiveMappings--;
7113	return VINF_SUCCESS;
7114	}
7115	#endif
7116
7117
7118	/**
7119	* Rollbacks mappings, releasing page locks and such.
7120	*
7121	* The caller shall only call this after checking cActiveMappings.
7122	*
7123	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7124	*/
7125	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT
7126	{
7127	Assert(pVCpu->iem.s.cActiveMappings > 0);
7128
7129	uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
7130	while (iMemMap-- > 0)
7131	{
7132	uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
7133	if (fAccess != IEM_ACCESS_INVALID)
7134	{
7135	AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
7136	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
7137	if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_NOT_LOCKED)))
7138	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
7139	AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
7140	("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
7141	iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
7142	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
7143	pVCpu->iem.s.cActiveMappings--;
7144	}
7145	}
7146	}
7147
7148
7149	/*
7150	* Instantiate R/W templates.
7151	*/
7152	#define TMPL_MEM_WITH_STACK
7153
7154	#define TMPL_MEM_TYPE uint8_t
7155	#define TMPL_MEM_FN_SUFF U8
7156	#define TMPL_MEM_FMT_TYPE "%#04x"
7157	#define TMPL_MEM_FMT_DESC "byte"
7158	#include "IEMAllMemRWTmpl.cpp.h"
7159
7160	#define TMPL_MEM_TYPE uint16_t
7161	#define TMPL_MEM_FN_SUFF U16
7162	#define TMPL_MEM_FMT_TYPE "%#06x"
7163	#define TMPL_MEM_FMT_DESC "word"
7164	#include "IEMAllMemRWTmpl.cpp.h"
7165
7166	#define TMPL_WITH_PUSH_SREG
7167	#define TMPL_MEM_TYPE uint32_t
7168	#define TMPL_MEM_FN_SUFF U32
7169	#define TMPL_MEM_FMT_TYPE "%#010x"
7170	#define TMPL_MEM_FMT_DESC "dword"
7171	#include "IEMAllMemRWTmpl.cpp.h"
7172	#undef TMPL_WITH_PUSH_SREG
7173
7174	#define TMPL_MEM_TYPE uint64_t
7175	#define TMPL_MEM_FN_SUFF U64
7176	#define TMPL_MEM_FMT_TYPE "%#018RX64"
7177	#define TMPL_MEM_FMT_DESC "qword"
7178	#include "IEMAllMemRWTmpl.cpp.h"
7179
7180	#undef TMPL_MEM_WITH_STACK
7181
7182	#define TMPL_MEM_TYPE uint64_t
7183	#define TMPL_MEM_TYPE_ALIGN (sizeof(uint64_t) * 2 - 1)
7184	#define TMPL_MEM_FN_SUFF U64AlignedU128
7185	#define TMPL_MEM_FMT_TYPE "%#018RX64"
7186	#define TMPL_MEM_FMT_DESC "qword"
7187	#include "IEMAllMemRWTmpl.cpp.h"
7188
7189	/* See IEMAllMemRWTmplInline.cpp.h */
7190	#define TMPL_MEM_BY_REF
7191
7192	#define TMPL_MEM_TYPE RTFLOAT80U
7193	#define TMPL_MEM_TYPE_ALIGN (sizeof(uint64_t) - 1)
7194	#define TMPL_MEM_FN_SUFF R80
7195	#define TMPL_MEM_FMT_TYPE "%.10Rhxs"
7196	#define TMPL_MEM_FMT_DESC "tword"
7197	#include "IEMAllMemRWTmpl.cpp.h"
7198
7199	#define TMPL_MEM_TYPE RTPBCD80U
7200	#define TMPL_MEM_TYPE_ALIGN (sizeof(uint64_t) - 1) /** @todo testcase: 80-bit BCD alignment */
7201	#define TMPL_MEM_FN_SUFF D80
7202	#define TMPL_MEM_FMT_TYPE "%.10Rhxs"
7203	#define TMPL_MEM_FMT_DESC "tword"
7204	#include "IEMAllMemRWTmpl.cpp.h"
7205
7206	#define TMPL_MEM_TYPE RTUINT128U
7207	#define TMPL_MEM_TYPE_ALIGN (sizeof(RTUINT128U) - 1)
7208	#define TMPL_MEM_FN_SUFF U128
7209	#define TMPL_MEM_FMT_TYPE "%.16Rhxs"
7210	#define TMPL_MEM_FMT_DESC "dqword"
7211	#include "IEMAllMemRWTmpl.cpp.h"
7212
7213	#define TMPL_MEM_TYPE RTUINT128U
7214	#define TMPL_MEM_TYPE_ALIGN (sizeof(RTUINT128U) - 1)
7215	#define TMPL_MEM_MAP_FLAGS_ADD (IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE)
7216	#define TMPL_MEM_FN_SUFF U128AlignedSse
7217	#define TMPL_MEM_FMT_TYPE "%.16Rhxs"
7218	#define TMPL_MEM_FMT_DESC "dqword"
7219	#include "IEMAllMemRWTmpl.cpp.h"
7220
7221	#define TMPL_MEM_TYPE RTUINT128U
7222	#define TMPL_MEM_TYPE_ALIGN 0
7223	#define TMPL_MEM_FN_SUFF U128NoAc
7224	#define TMPL_MEM_FMT_TYPE "%.16Rhxs"
7225	#define TMPL_MEM_FMT_DESC "dqword"
7226	#include "IEMAllMemRWTmpl.cpp.h"
7227
7228	#define TMPL_MEM_TYPE RTUINT256U
7229	#define TMPL_MEM_TYPE_ALIGN 0
7230	#define TMPL_MEM_FN_SUFF U256NoAc
7231	#define TMPL_MEM_FMT_TYPE "%.32Rhxs"
7232	#define TMPL_MEM_FMT_DESC "qqword"
7233	#include "IEMAllMemRWTmpl.cpp.h"
7234
7235	/**
7236	* Fetches a data dword and zero extends it to a qword.
7237	*
7238	* @returns Strict VBox status code.
7239	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7240	* @param pu64Dst Where to return the qword.
7241	* @param iSegReg The index of the segment register to use for
7242	* this access. The base and limits are checked.
7243	* @param GCPtrMem The address of the guest memory.
7244	*/
7245	VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7246	{
7247	/* The lazy approach for now... */
7248	uint8_t bUnmapInfo;
7249	uint32_t const *pu32Src;
7250	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, &bUnmapInfo, sizeof(pu32Src), iSegReg, GCPtrMem,
7251	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
7252	if (rc == VINF_SUCCESS)
7253	{
7254	pu64Dst = pu32Src;
7255	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7256	Log(("IEM RD dword %d\|%RGv: %#010RX64\n", iSegReg, GCPtrMem, *pu64Dst));
7257	}
7258	return rc;
7259	}
7260
7261
7262	#ifdef SOME_UNUSED_FUNCTION
7263	/**
7264	* Fetches a data dword and sign extends it to a qword.
7265	*
7266	* @returns Strict VBox status code.
7267	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7268	* @param pu64Dst Where to return the sign extended value.
7269	* @param iSegReg The index of the segment register to use for
7270	* this access. The base and limits are checked.
7271	* @param GCPtrMem The address of the guest memory.
7272	*/
7273	VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7274	{
7275	/* The lazy approach for now... */
7276	uint8_t bUnmapInfo;
7277	int32_t const *pi32Src;
7278	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pi32Src, &bUnmapInfo, sizeof(pi32Src), iSegReg, GCPtrMem,
7279	IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1);
7280	if (rc == VINF_SUCCESS)
7281	{
7282	pu64Dst = pi32Src;
7283	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7284	Log(("IEM RD dword %d\|%RGv: %#010x\n", iSegReg, GCPtrMem, (uint32_t)*pu64Dst));
7285	}
7286	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
7287	else
7288	*pu64Dst = 0;
7289	#endif
7290	return rc;
7291	}
7292	#endif
7293
7294
7295	/**
7296	* Fetches a data oword (octo word) at an aligned address, generally AVX
7297	* related.
7298	*
7299	* Raises \#GP(0) if not aligned.
7300	*
7301	* @returns Strict VBox status code.
7302	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7303	* @param pu256Dst Where to return the qword.
7304	* @param iSegReg The index of the segment register to use for
7305	* this access. The base and limits are checked.
7306	* @param GCPtrMem The address of the guest memory.
7307	*/
7308	VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7309	{
7310	/* The lazy approach for now... */
7311	uint8_t bUnmapInfo;
7312	PCRTUINT256U pu256Src;
7313	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, &bUnmapInfo, sizeof(pu256Src), iSegReg, GCPtrMem,
7314	IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7315	if (rc == VINF_SUCCESS)
7316	{
7317	pu256Dst->au64[0] = pu256Src->au64[0];
7318	pu256Dst->au64[1] = pu256Src->au64[1];
7319	pu256Dst->au64[2] = pu256Src->au64[2];
7320	pu256Dst->au64[3] = pu256Src->au64[3];
7321	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7322	Log(("IEM RD qqword %d\|%RGv: %.32Rhxs\n", iSegReg, GCPtrMem, pu256Dst));
7323	}
7324	return rc;
7325	}
7326
7327
7328	#ifdef IEM_WITH_SETJMP
7329	/**
7330	* Fetches a data oword (octo word) at an aligned address, generally AVX
7331	* related, longjmp on error.
7332	*
7333	* Raises \#GP(0) if not aligned.
7334	*
7335	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7336	* @param pu256Dst Where to return the qword.
7337	* @param iSegReg The index of the segment register to use for
7338	* this access. The base and limits are checked.
7339	* @param GCPtrMem The address of the guest memory.
7340	*/
7341	void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg,
7342	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
7343	{
7344	/* The lazy approach for now... */
7345	uint8_t bUnmapInfo;
7346	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, &bUnmapInfo, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7347	(sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7348	pu256Dst->au64[0] = pu256Src->au64[0];
7349	pu256Dst->au64[1] = pu256Src->au64[1];
7350	pu256Dst->au64[2] = pu256Src->au64[2];
7351	pu256Dst->au64[3] = pu256Src->au64[3];
7352	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7353	Log(("IEM RD qqword %d\|%RGv: %.32Rhxs\n", iSegReg, GCPtrMem, pu256Dst));
7354	}
7355	#endif
7356
7357
7358
7359	/**
7360	* Fetches a descriptor register (lgdt, lidt).
7361	*
7362	* @returns Strict VBox status code.
7363	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7364	* @param pcbLimit Where to return the limit.
7365	* @param pGCPtrBase Where to return the base.
7366	* @param iSegReg The index of the segment register to use for
7367	* this access. The base and limits are checked.
7368	* @param GCPtrMem The address of the guest memory.
7369	* @param enmOpSize The effective operand size.
7370	*/
7371	VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
7372	RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT
7373	{
7374	/*
7375	* Just like SIDT and SGDT, the LIDT and LGDT instructions are a
7376	* little special:
7377	* - The two reads are done separately.
7378	* - Operand size override works in 16-bit and 32-bit code, but 64-bit.
7379	* - We suspect the 386 to actually commit the limit before the base in
7380	* some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
7381	* don't try emulate this eccentric behavior, because it's not well
7382	* enough understood and rather hard to trigger.
7383	* - The 486 seems to do a dword limit read when the operand size is 32-bit.
7384	*/
7385	VBOXSTRICTRC rcStrict;
7386	if (IEM_IS_64BIT_CODE(pVCpu))
7387	{
7388	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7389	if (rcStrict == VINF_SUCCESS)
7390	rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
7391	}
7392	else
7393	{
7394	uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
7395	if (enmOpSize == IEMMODE_32BIT)
7396	{
7397	if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
7398	{
7399	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7400	if (rcStrict == VINF_SUCCESS)
7401	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7402	}
7403	else
7404	{
7405	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
7406	if (rcStrict == VINF_SUCCESS)
7407	{
7408	*pcbLimit = (uint16_t)uTmp;
7409	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7410	}
7411	}
7412	if (rcStrict == VINF_SUCCESS)
7413	*pGCPtrBase = uTmp;
7414	}
7415	else
7416	{
7417	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7418	if (rcStrict == VINF_SUCCESS)
7419	{
7420	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7421	if (rcStrict == VINF_SUCCESS)
7422	*pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
7423	}
7424	}
7425	}
7426	return rcStrict;
7427	}
7428
7429
7430	/**
7431	* Stores a data dqword, SSE aligned.
7432	*
7433	* @returns Strict VBox status code.
7434	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7435	* @param iSegReg The index of the segment register to use for
7436	* this access. The base and limits are checked.
7437	* @param GCPtrMem The address of the guest memory.
7438	* @param u128Value The value to store.
7439	*/
7440	VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7441	{
7442	/* The lazy approach for now... */
7443	uint8_t bUnmapInfo;
7444	PRTUINT128U pu128Dst;
7445	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, &bUnmapInfo, sizeof(pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7446	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7447	if (rc == VINF_SUCCESS)
7448	{
7449	pu128Dst->au64[0] = u128Value.au64[0];
7450	pu128Dst->au64[1] = u128Value.au64[1];
7451	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7452	Log5(("IEM WR dqword %d\|%RGv: %.16Rhxs\n", iSegReg, GCPtrMem, pu128Dst));
7453	}
7454	return rc;
7455	}
7456
7457
7458	#ifdef IEM_WITH_SETJMP
7459	/**
7460	* Stores a data dqword, SSE aligned.
7461	*
7462	* @returns Strict VBox status code.
7463	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7464	* @param iSegReg The index of the segment register to use for
7465	* this access. The base and limits are checked.
7466	* @param GCPtrMem The address of the guest memory.
7467	* @param u128Value The value to store.
7468	*/
7469	void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem,
7470	RTUINT128U u128Value) IEM_NOEXCEPT_MAY_LONGJMP
7471	{
7472	/* The lazy approach for now... */
7473	uint8_t bUnmapInfo;
7474	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, &bUnmapInfo, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7475	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7476	pu128Dst->au64[0] = u128Value.au64[0];
7477	pu128Dst->au64[1] = u128Value.au64[1];
7478	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7479	Log5(("IEM WR dqword %d\|%RGv: %.16Rhxs\n", iSegReg, GCPtrMem, pu128Dst));
7480	}
7481	#endif
7482
7483
7484	/**
7485	* Stores a data dqword.
7486	*
7487	* @returns Strict VBox status code.
7488	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7489	* @param iSegReg The index of the segment register to use for
7490	* this access. The base and limits are checked.
7491	* @param GCPtrMem The address of the guest memory.
7492	* @param pu256Value Pointer to the value to store.
7493	*/
7494	VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7495	{
7496	/* The lazy approach for now... */
7497	uint8_t bUnmapInfo;
7498	PRTUINT256U pu256Dst;
7499	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, &bUnmapInfo, sizeof(pu256Dst), iSegReg, GCPtrMem,
7500	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7501	if (rc == VINF_SUCCESS)
7502	{
7503	pu256Dst->au64[0] = pu256Value->au64[0];
7504	pu256Dst->au64[1] = pu256Value->au64[1];
7505	pu256Dst->au64[2] = pu256Value->au64[2];
7506	pu256Dst->au64[3] = pu256Value->au64[3];
7507	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7508	Log5(("IEM WR qqword %d\|%RGv: %.32Rhxs\n", iSegReg, GCPtrMem, pu256Dst));
7509	}
7510	return rc;
7511	}
7512
7513
7514	#ifdef IEM_WITH_SETJMP
7515	/**
7516	* Stores a data dqword, longjmp on error.
7517	*
7518	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7519	* @param iSegReg The index of the segment register to use for
7520	* this access. The base and limits are checked.
7521	* @param GCPtrMem The address of the guest memory.
7522	* @param pu256Value Pointer to the value to store.
7523	*/
7524	void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) IEM_NOEXCEPT_MAY_LONGJMP
7525	{
7526	/* The lazy approach for now... */
7527	uint8_t bUnmapInfo;
7528	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, &bUnmapInfo, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7529	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7530	pu256Dst->au64[0] = pu256Value->au64[0];
7531	pu256Dst->au64[1] = pu256Value->au64[1];
7532	pu256Dst->au64[2] = pu256Value->au64[2];
7533	pu256Dst->au64[3] = pu256Value->au64[3];
7534	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7535	Log5(("IEM WR qqword %d\|%RGv: %.32Rhxs\n", iSegReg, GCPtrMem, pu256Dst));
7536	}
7537	#endif
7538
7539
7540	/**
7541	* Stores a data dqword, AVX \#GP(0) aligned.
7542	*
7543	* @returns Strict VBox status code.
7544	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7545	* @param iSegReg The index of the segment register to use for
7546	* this access. The base and limits are checked.
7547	* @param GCPtrMem The address of the guest memory.
7548	* @param pu256Value Pointer to the value to store.
7549	*/
7550	VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7551	{
7552	/* The lazy approach for now... */
7553	uint8_t bUnmapInfo;
7554	PRTUINT256U pu256Dst;
7555	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, &bUnmapInfo, sizeof(pu256Dst), iSegReg, GCPtrMem,
7556	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7557	if (rc == VINF_SUCCESS)
7558	{
7559	pu256Dst->au64[0] = pu256Value->au64[0];
7560	pu256Dst->au64[1] = pu256Value->au64[1];
7561	pu256Dst->au64[2] = pu256Value->au64[2];
7562	pu256Dst->au64[3] = pu256Value->au64[3];
7563	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7564	Log5(("IEM WR qqword %d\|%RGv: %.32Rhxs\n", iSegReg, GCPtrMem, pu256Dst));
7565	}
7566	return rc;
7567	}
7568
7569
7570	#ifdef IEM_WITH_SETJMP
7571	/**
7572	* Stores a data dqword, AVX aligned.
7573	*
7574	* @returns Strict VBox status code.
7575	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7576	* @param iSegReg The index of the segment register to use for
7577	* this access. The base and limits are checked.
7578	* @param GCPtrMem The address of the guest memory.
7579	* @param pu256Value Pointer to the value to store.
7580	*/
7581	void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem,
7582	PCRTUINT256U pu256Value) IEM_NOEXCEPT_MAY_LONGJMP
7583	{
7584	/* The lazy approach for now... */
7585	uint8_t bUnmapInfo;
7586	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, &bUnmapInfo, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7587	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7588	pu256Dst->au64[0] = pu256Value->au64[0];
7589	pu256Dst->au64[1] = pu256Value->au64[1];
7590	pu256Dst->au64[2] = pu256Value->au64[2];
7591	pu256Dst->au64[3] = pu256Value->au64[3];
7592	iemMemCommitAndUnmapJmp(pVCpu, bUnmapInfo);
7593	Log5(("IEM WR qqword %d\|%RGv: %.32Rhxs\n", iSegReg, GCPtrMem, pu256Dst));
7594	}
7595	#endif
7596
7597
7598	/**
7599	* Stores a descriptor register (sgdt, sidt).
7600	*
7601	* @returns Strict VBox status code.
7602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7603	* @param cbLimit The limit.
7604	* @param GCPtrBase The base address.
7605	* @param iSegReg The index of the segment register to use for
7606	* this access. The base and limits are checked.
7607	* @param GCPtrMem The address of the guest memory.
7608	*/
7609	VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7610	{
7611	/*
7612	* The SIDT and SGDT instructions actually stores the data using two
7613	* independent writes (see bs3CpuBasic2_sidt_sgdt_One). The instructions
7614	* does not respond to opsize prefixes.
7615	*/
7616	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
7617	if (rcStrict == VINF_SUCCESS)
7618	{
7619	if (IEM_IS_16BIT_CODE(pVCpu))
7620	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
7621	IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
7622	? (uint32_t)GCPtrBase \| UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
7623	else if (IEM_IS_32BIT_CODE(pVCpu))
7624	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
7625	else
7626	rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
7627	}
7628	return rcStrict;
7629	}
7630
7631
7632	/**
7633	* Begin a special stack push (used by interrupt, exceptions and such).
7634	*
7635	* This will raise \#SS or \#PF if appropriate.
7636	*
7637	* @returns Strict VBox status code.
7638	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7639	* @param cbMem The number of bytes to push onto the stack.
7640	* @param cbAlign The alignment mask (7, 3, 1).
7641	* @param ppvMem Where to return the pointer to the stack memory.
7642	* As with the other memory functions this could be
7643	* direct access or bounce buffered access, so
7644	* don't commit register until the commit call
7645	* succeeds.
7646	* @param pbUnmapInfo Where to store unmap info for
7647	* iemMemStackPushCommitSpecial.
7648	* @param puNewRsp Where to return the new RSP value. This must be
7649	* passed unchanged to
7650	* iemMemStackPushCommitSpecial().
7651	*/
7652	VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
7653	void *ppvMem, uint8_t pbUnmapInfo, uint64_t *puNewRsp) RT_NOEXCEPT
7654	{
7655	Assert(cbMem < UINT8_MAX);
7656	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
7657	return iemMemMap(pVCpu, ppvMem, pbUnmapInfo, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W, cbAlign);
7658	}
7659
7660
7661	/**
7662	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
7663	*
7664	* This will update the rSP.
7665	*
7666	* @returns Strict VBox status code.
7667	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7668	* @param bUnmapInfo Unmap info set by iemMemStackPushBeginSpecial.
7669	* @param uNewRsp The new RSP value returned by
7670	* iemMemStackPushBeginSpecial().
7671	*/
7672	VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, uint8_t bUnmapInfo, uint64_t uNewRsp) RT_NOEXCEPT
7673	{
7674	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7675	if (rcStrict == VINF_SUCCESS)
7676	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7677	return rcStrict;
7678	}
7679
7680
7681	/**
7682	* Begin a special stack pop (used by iret, retf and such).
7683	*
7684	* This will raise \#SS or \#PF if appropriate.
7685	*
7686	* @returns Strict VBox status code.
7687	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7688	* @param cbMem The number of bytes to pop from the stack.
7689	* @param cbAlign The alignment mask (7, 3, 1).
7690	* @param ppvMem Where to return the pointer to the stack memory.
7691	* @param pbUnmapInfo Where to store unmap info for
7692	* iemMemStackPopDoneSpecial.
7693	* @param puNewRsp Where to return the new RSP value. This must be
7694	* assigned to CPUMCTX::rsp manually some time
7695	* after iemMemStackPopDoneSpecial() has been
7696	* called.
7697	*/
7698	VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
7699	void const *ppvMem, uint8_t pbUnmapInfo, uint64_t *puNewRsp) RT_NOEXCEPT
7700	{
7701	Assert(cbMem < UINT8_MAX);
7702	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
7703	return iemMemMap(pVCpu, (void **)ppvMem, pbUnmapInfo, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign);
7704	}
7705
7706
7707	/**
7708	* Continue a special stack pop (used by iret and retf), for the purpose of
7709	* retrieving a new stack pointer.
7710	*
7711	* This will raise \#SS or \#PF if appropriate.
7712	*
7713	* @returns Strict VBox status code.
7714	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7715	* @param off Offset from the top of the stack. This is zero
7716	* except in the retf case.
7717	* @param cbMem The number of bytes to pop from the stack.
7718	* @param ppvMem Where to return the pointer to the stack memory.
7719	* @param pbUnmapInfo Where to store unmap info for
7720	* iemMemStackPopDoneSpecial.
7721	* @param uCurNewRsp The current uncommitted RSP value. (No need to
7722	* return this because all use of this function is
7723	* to retrieve a new value and anything we return
7724	* here would be discarded.)
7725	*/
7726	VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t off, size_t cbMem,
7727	void const *ppvMem, uint8_t pbUnmapInfo, uint64_t uCurNewRsp) RT_NOEXCEPT
7728	{
7729	Assert(cbMem < UINT8_MAX);
7730
7731	/* The essense of iemRegGetRspForPopEx and friends: / /* @todo put this into a inlined function? */
7732	RTGCPTR GCPtrTop;
7733	if (IEM_IS_64BIT_CODE(pVCpu))
7734	GCPtrTop = uCurNewRsp;
7735	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
7736	GCPtrTop = (uint32_t)uCurNewRsp;
7737	else
7738	GCPtrTop = (uint16_t)uCurNewRsp;
7739
7740	return iemMemMap(pVCpu, (void **)ppvMem, pbUnmapInfo, cbMem, X86_SREG_SS, GCPtrTop + off, IEM_ACCESS_STACK_R,
7741	0 /* checked in iemMemStackPopBeginSpecial */);
7742	}
7743
7744
7745	/**
7746	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
7747	* iemMemStackPopContinueSpecial).
7748	*
7749	* The caller will manually commit the rSP.
7750	*
7751	* @returns Strict VBox status code.
7752	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7753	* @param bUnmapInfo Unmap information returned by
7754	* iemMemStackPopBeginSpecial() or
7755	* iemMemStackPopContinueSpecial().
7756	*/
7757	VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, uint8_t bUnmapInfo) RT_NOEXCEPT
7758	{
7759	return iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7760	}
7761
7762
7763	/**
7764	* Fetches a system table byte.
7765	*
7766	* @returns Strict VBox status code.
7767	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7768	* @param pbDst Where to return the byte.
7769	* @param iSegReg The index of the segment register to use for
7770	* this access. The base and limits are checked.
7771	* @param GCPtrMem The address of the guest memory.
7772	*/
7773	VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7774	{
7775	/* The lazy approach for now... */
7776	uint8_t bUnmapInfo;
7777	uint8_t const *pbSrc;
7778	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pbSrc, &bUnmapInfo, sizeof(pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
7779	if (rc == VINF_SUCCESS)
7780	{
7781	pbDst = pbSrc;
7782	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7783	}
7784	return rc;
7785	}
7786
7787
7788	/**
7789	* Fetches a system table word.
7790	*
7791	* @returns Strict VBox status code.
7792	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7793	* @param pu16Dst Where to return the word.
7794	* @param iSegReg The index of the segment register to use for
7795	* this access. The base and limits are checked.
7796	* @param GCPtrMem The address of the guest memory.
7797	*/
7798	VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7799	{
7800	/* The lazy approach for now... */
7801	uint8_t bUnmapInfo;
7802	uint16_t const *pu16Src;
7803	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, &bUnmapInfo, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
7804	if (rc == VINF_SUCCESS)
7805	{
7806	pu16Dst = pu16Src;
7807	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7808	}
7809	return rc;
7810	}
7811
7812
7813	/**
7814	* Fetches a system table dword.
7815	*
7816	* @returns Strict VBox status code.
7817	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7818	* @param pu32Dst Where to return the dword.
7819	* @param iSegReg The index of the segment register to use for
7820	* this access. The base and limits are checked.
7821	* @param GCPtrMem The address of the guest memory.
7822	*/
7823	VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7824	{
7825	/* The lazy approach for now... */
7826	uint8_t bUnmapInfo;
7827	uint32_t const *pu32Src;
7828	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, &bUnmapInfo, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
7829	if (rc == VINF_SUCCESS)
7830	{
7831	pu32Dst = pu32Src;
7832	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7833	}
7834	return rc;
7835	}
7836
7837
7838	/**
7839	* Fetches a system table qword.
7840	*
7841	* @returns Strict VBox status code.
7842	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7843	* @param pu64Dst Where to return the qword.
7844	* @param iSegReg The index of the segment register to use for
7845	* this access. The base and limits are checked.
7846	* @param GCPtrMem The address of the guest memory.
7847	*/
7848	VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7849	{
7850	/* The lazy approach for now... */
7851	uint8_t bUnmapInfo;
7852	uint64_t const *pu64Src;
7853	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, &bUnmapInfo, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
7854	if (rc == VINF_SUCCESS)
7855	{
7856	pu64Dst = pu64Src;
7857	rc = iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
7858	}
7859	return rc;
7860	}
7861
7862
7863	/**
7864	* Fetches a descriptor table entry with caller specified error code.
7865	*
7866	* @returns Strict VBox status code.
7867	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7868	* @param pDesc Where to return the descriptor table entry.
7869	* @param uSel The selector which table entry to fetch.
7870	* @param uXcpt The exception to raise on table lookup error.
7871	* @param uErrorCode The error code associated with the exception.
7872	*/
7873	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
7874	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT
7875	{
7876	AssertPtr(pDesc);
7877	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
7878
7879	/** @todo did the 286 require all 8 bytes to be accessible? */
7880	/*
7881	* Get the selector table base and check bounds.
7882	*/
7883	RTGCPTR GCPtrBase;
7884	if (uSel & X86_SEL_LDT)
7885	{
7886	if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
7887	\|\| (uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
7888	{
7889	LogEx(LOG_GROUP_IEM, ("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
7890	uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
7891	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
7892	uErrorCode, 0);
7893	}
7894
7895	Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
7896	GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
7897	}
7898	else
7899	{
7900	if ((uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
7901	{
7902	LogEx(LOG_GROUP_IEM, ("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
7903	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
7904	uErrorCode, 0);
7905	}
7906	GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
7907	}
7908
7909	/*
7910	* Read the legacy descriptor and maybe the long mode extensions if
7911	* required.
7912	*/
7913	VBOXSTRICTRC rcStrict;
7914	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
7915	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
7916	else
7917	{
7918	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
7919	if (rcStrict == VINF_SUCCESS)
7920	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
7921	if (rcStrict == VINF_SUCCESS)
7922	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
7923	if (rcStrict == VINF_SUCCESS)
7924	pDesc->Legacy.au16[3] = 0;
7925	else
7926	return rcStrict;
7927	}
7928
7929	if (rcStrict == VINF_SUCCESS)
7930	{
7931	if ( !IEM_IS_LONG_MODE(pVCpu)
7932	\|\| pDesc->Legacy.Gen.u1DescType)
7933	pDesc->Long.au64[1] = 0;
7934	else if ( (uint32_t)(uSel \| X86_SEL_RPL_LDT) + 8
7935	<= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
7936	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel \| X86_SEL_RPL_LDT) + 1);
7937	else
7938	{
7939	LogEx(LOG_GROUP_IEM,("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
7940	/** @todo is this the right exception? */
7941	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
7942	}
7943	}
7944	return rcStrict;
7945	}
7946
7947
7948	/**
7949	* Fetches a descriptor table entry.
7950	*
7951	* @returns Strict VBox status code.
7952	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7953	* @param pDesc Where to return the descriptor table entry.
7954	* @param uSel The selector which table entry to fetch.
7955	* @param uXcpt The exception to raise on table lookup error.
7956	*/
7957	VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT
7958	{
7959	return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
7960	}
7961
7962
7963	/**
7964	* Marks the selector descriptor as accessed (only non-system descriptors).
7965	*
7966	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
7967	* will therefore skip the limit checks.
7968	*
7969	* @returns Strict VBox status code.
7970	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7971	* @param uSel The selector.
7972	*/
7973	VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
7974	{
7975	/*
7976	* Get the selector table base and calculate the entry address.
7977	*/
7978	RTGCPTR GCPtr = uSel & X86_SEL_LDT
7979	? pVCpu->cpum.GstCtx.ldtr.u64Base
7980	: pVCpu->cpum.GstCtx.gdtr.pGdt;
7981	GCPtr += uSel & X86_SEL_MASK;
7982
7983	/*
7984	* ASMAtomicBitSet will assert if the address is misaligned, so do some
7985	* ugly stuff to avoid this. This will make sure it's an atomic access
7986	* as well more or less remove any question about 8-bit or 32-bit accesss.
7987	*/
7988	VBOXSTRICTRC rcStrict;
7989	uint8_t bUnmapInfo;
7990	uint32_t volatile *pu32;
7991	if ((GCPtr & 3) == 0)
7992	{
7993	/* The normal case, map the 32-bit bits around the accessed bit (40). */
7994	GCPtr += 2 + 2;
7995	rcStrict = iemMemMap(pVCpu, (void **)&pu32, &bUnmapInfo, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
7996	if (rcStrict != VINF_SUCCESS)
7997	return rcStrict;
7998	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
7999	}
8000	else
8001	{
8002	/* The misaligned GDT/LDT case, map the whole thing. */
8003	rcStrict = iemMemMap(pVCpu, (void **)&pu32, &bUnmapInfo, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8004	if (rcStrict != VINF_SUCCESS)
8005	return rcStrict;
8006	switch ((uintptr_t)pu32 & 3)
8007	{
8008	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
8009	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
8010	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
8011	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
8012	}
8013	}
8014
8015	return iemMemCommitAndUnmap(pVCpu, bUnmapInfo);
8016	}
8017
8018
8019	#undef LOG_GROUP
8020	#define LOG_GROUP LOG_GROUP_IEM
8021
8022	/** @} */
8023
8024	/** @name Opcode Helpers.
8025	* @{
8026	*/
8027
8028	/**
8029	* Calculates the effective address of a ModR/M memory operand.
8030	*
8031	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8032	*
8033	* @return Strict VBox status code.
8034	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8035	* @param bRm The ModRM byte.
8036	* @param cbImmAndRspOffset - First byte: The size of any immediate
8037	* following the effective address opcode bytes
8038	* (only for RIP relative addressing).
8039	* - Second byte: RSP displacement (for POP [ESP]).
8040	* @param pGCPtrEff Where to return the effective address.
8041	*/
8042	VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset, PRTGCPTR pGCPtrEff) RT_NOEXCEPT
8043	{
8044	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8045	# define SET_SS_DEF() \
8046	do \
8047	{ \
8048	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8049	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8050	} while (0)
8051
8052	if (!IEM_IS_64BIT_CODE(pVCpu))
8053	{
8054	/** @todo Check the effective address size crap! */
8055	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8056	{
8057	uint16_t u16EffAddr;
8058
8059	/* Handle the disp16 form with no registers first. */
8060	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8061	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8062	else
8063	{
8064	/* Get the displacment. */
8065	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8066	{
8067	case 0: u16EffAddr = 0; break;
8068	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8069	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8070	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8071	}
8072
8073	/* Add the base and index registers to the disp. */
8074	switch (bRm & X86_MODRM_RM_MASK)
8075	{
8076	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8077	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8078	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8079	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8080	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8081	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8082	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8083	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8084	}
8085	}
8086
8087	*pGCPtrEff = u16EffAddr;
8088	}
8089	else
8090	{
8091	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8092	uint32_t u32EffAddr;
8093
8094	/* Handle the disp32 form with no registers first. */
8095	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8096	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8097	else
8098	{
8099	/* Get the register (or SIB) value. */
8100	switch ((bRm & X86_MODRM_RM_MASK))
8101	{
8102	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8103	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8104	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8105	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8106	case 4: /* SIB */
8107	{
8108	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8109
8110	/* Get the index and scale it. */
8111	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8112	{
8113	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8114	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8115	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8116	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8117	case 4: u32EffAddr = 0; /none / break;
8118	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8119	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8120	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8121	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8122	}
8123	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8124
8125	/* add base */
8126	switch (bSib & X86_SIB_BASE_MASK)
8127	{
8128	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8129	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8130	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8131	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8132	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
8133	case 5:
8134	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8135	{
8136	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8137	SET_SS_DEF();
8138	}
8139	else
8140	{
8141	uint32_t u32Disp;
8142	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8143	u32EffAddr += u32Disp;
8144	}
8145	break;
8146	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8147	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8148	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8149	}
8150	break;
8151	}
8152	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8153	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8154	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8155	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8156	}
8157
8158	/* Get and add the displacement. */
8159	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8160	{
8161	case 0:
8162	break;
8163	case 1:
8164	{
8165	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8166	u32EffAddr += i8Disp;
8167	break;
8168	}
8169	case 2:
8170	{
8171	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8172	u32EffAddr += u32Disp;
8173	break;
8174	}
8175	default:
8176	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
8177	}
8178
8179	}
8180	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8181	*pGCPtrEff = u32EffAddr;
8182	}
8183	}
8184	else
8185	{
8186	uint64_t u64EffAddr;
8187
8188	/* Handle the rip+disp32 form with no registers first. */
8189	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8190	{
8191	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8192	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
8193	}
8194	else
8195	{
8196	/* Get the register (or SIB) value. */
8197	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
8198	{
8199	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8200	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8201	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8202	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8203	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8204	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8205	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8206	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8207	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8208	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8209	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8210	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8211	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8212	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8213	/* SIB */
8214	case 4:
8215	case 12:
8216	{
8217	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8218
8219	/* Get the index and scale it. */
8220	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
8221	{
8222	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8223	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8224	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8225	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8226	case 4: u64EffAddr = 0; /none / break;
8227	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8228	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8229	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8230	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8231	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8232	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8233	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8234	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8235	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8236	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8237	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8238	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8239	}
8240	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8241
8242	/* add base */
8243	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
8244	{
8245	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8246	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8247	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8248	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8249	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
8250	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8251	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8252	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8253	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8254	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8255	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8256	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8257	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8258	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8259	/* complicated encodings */
8260	case 5:
8261	case 13:
8262	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8263	{
8264	if (!pVCpu->iem.s.uRexB)
8265	{
8266	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8267	SET_SS_DEF();
8268	}
8269	else
8270	u64EffAddr += pVCpu->cpum.GstCtx.r13;
8271	}
8272	else
8273	{
8274	uint32_t u32Disp;
8275	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8276	u64EffAddr += (int32_t)u32Disp;
8277	}
8278	break;
8279	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8280	}
8281	break;
8282	}
8283	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8284	}
8285
8286	/* Get and add the displacement. */
8287	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8288	{
8289	case 0:
8290	break;
8291	case 1:
8292	{
8293	int8_t i8Disp;
8294	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8295	u64EffAddr += i8Disp;
8296	break;
8297	}
8298	case 2:
8299	{
8300	uint32_t u32Disp;
8301	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8302	u64EffAddr += (int32_t)u32Disp;
8303	break;
8304	}
8305	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
8306	}
8307
8308	}
8309
8310	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8311	*pGCPtrEff = u64EffAddr;
8312	else
8313	{
8314	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8315	*pGCPtrEff = u64EffAddr & UINT32_MAX;
8316	}
8317	}
8318
8319	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
8320	return VINF_SUCCESS;
8321	}
8322
8323
8324	#ifdef IEM_WITH_SETJMP
8325	/**
8326	* Calculates the effective address of a ModR/M memory operand.
8327	*
8328	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8329	*
8330	* May longjmp on internal error.
8331	*
8332	* @return The effective address.
8333	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8334	* @param bRm The ModRM byte.
8335	* @param cbImmAndRspOffset - First byte: The size of any immediate
8336	* following the effective address opcode bytes
8337	* (only for RIP relative addressing).
8338	* - Second byte: RSP displacement (for POP [ESP]).
8339	*/
8340	RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset) IEM_NOEXCEPT_MAY_LONGJMP
8341	{
8342	Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
8343	# define SET_SS_DEF() \
8344	do \
8345	{ \
8346	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8347	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8348	} while (0)
8349
8350	if (!IEM_IS_64BIT_CODE(pVCpu))
8351	{
8352	/** @todo Check the effective address size crap! */
8353	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8354	{
8355	uint16_t u16EffAddr;
8356
8357	/* Handle the disp16 form with no registers first. */
8358	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8359	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8360	else
8361	{
8362	/* Get the displacment. */
8363	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8364	{
8365	case 0: u16EffAddr = 0; break;
8366	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8367	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8368	default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */
8369	}
8370
8371	/* Add the base and index registers to the disp. */
8372	switch (bRm & X86_MODRM_RM_MASK)
8373	{
8374	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8375	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8376	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8377	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8378	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8379	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8380	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8381	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8382	}
8383	}
8384
8385	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
8386	return u16EffAddr;
8387	}
8388
8389	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8390	uint32_t u32EffAddr;
8391
8392	/* Handle the disp32 form with no registers first. */
8393	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8394	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8395	else
8396	{
8397	/* Get the register (or SIB) value. */
8398	switch ((bRm & X86_MODRM_RM_MASK))
8399	{
8400	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8401	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8402	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8403	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8404	case 4: /* SIB */
8405	{
8406	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8407
8408	/* Get the index and scale it. */
8409	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8410	{
8411	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8412	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8413	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8414	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8415	case 4: u32EffAddr = 0; /none / break;
8416	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8417	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8418	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8419	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
8420	}
8421	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8422
8423	/* add base */
8424	switch (bSib & X86_SIB_BASE_MASK)
8425	{
8426	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8427	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8428	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8429	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8430	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
8431	case 5:
8432	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8433	{
8434	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8435	SET_SS_DEF();
8436	}
8437	else
8438	{
8439	uint32_t u32Disp;
8440	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8441	u32EffAddr += u32Disp;
8442	}
8443	break;
8444	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8445	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8446	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
8447	}
8448	break;
8449	}
8450	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8451	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8452	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8453	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
8454	}
8455
8456	/* Get and add the displacement. */
8457	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8458	{
8459	case 0:
8460	break;
8461	case 1:
8462	{
8463	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8464	u32EffAddr += i8Disp;
8465	break;
8466	}
8467	case 2:
8468	{
8469	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8470	u32EffAddr += u32Disp;
8471	break;
8472	}
8473	default:
8474	AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */
8475	}
8476	}
8477
8478	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8479	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
8480	return u32EffAddr;
8481	}
8482
8483	uint64_t u64EffAddr;
8484
8485	/* Handle the rip+disp32 form with no registers first. */
8486	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8487	{
8488	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8489	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
8490	}
8491	else
8492	{
8493	/* Get the register (or SIB) value. */
8494	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
8495	{
8496	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8497	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8498	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8499	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8500	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8501	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8502	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8503	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8504	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8505	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8506	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8507	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8508	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8509	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8510	/* SIB */
8511	case 4:
8512	case 12:
8513	{
8514	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8515
8516	/* Get the index and scale it. */
8517	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
8518	{
8519	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8520	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8521	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8522	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8523	case 4: u64EffAddr = 0; /none / break;
8524	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8525	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8526	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8527	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8528	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8529	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8530	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8531	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8532	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8533	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8534	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8535	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
8536	}
8537	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8538
8539	/* add base */
8540	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
8541	{
8542	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8543	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8544	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8545	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8546	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
8547	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8548	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8549	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8550	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8551	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8552	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8553	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8554	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8555	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8556	/* complicated encodings */
8557	case 5:
8558	case 13:
8559	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8560	{
8561	if (!pVCpu->iem.s.uRexB)
8562	{
8563	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8564	SET_SS_DEF();
8565	}
8566	else
8567	u64EffAddr += pVCpu->cpum.GstCtx.r13;
8568	}
8569	else
8570	{
8571	uint32_t u32Disp;
8572	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8573	u64EffAddr += (int32_t)u32Disp;
8574	}
8575	break;
8576	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
8577	}
8578	break;
8579	}
8580	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
8581	}
8582
8583	/* Get and add the displacement. */
8584	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8585	{
8586	case 0:
8587	break;
8588	case 1:
8589	{
8590	int8_t i8Disp;
8591	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8592	u64EffAddr += i8Disp;
8593	break;
8594	}
8595	case 2:
8596	{
8597	uint32_t u32Disp;
8598	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8599	u64EffAddr += (int32_t)u32Disp;
8600	break;
8601	}
8602	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
8603	}
8604
8605	}
8606
8607	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8608	{
8609	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
8610	return u64EffAddr;
8611	}
8612	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8613	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
8614	return u64EffAddr & UINT32_MAX;
8615	}
8616	#endif /* IEM_WITH_SETJMP */
8617
8618
8619	/**
8620	* Calculates the effective address of a ModR/M memory operand, extended version
8621	* for use in the recompilers.
8622	*
8623	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8624	*
8625	* @return Strict VBox status code.
8626	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8627	* @param bRm The ModRM byte.
8628	* @param cbImmAndRspOffset - First byte: The size of any immediate
8629	* following the effective address opcode bytes
8630	* (only for RIP relative addressing).
8631	* - Second byte: RSP displacement (for POP [ESP]).
8632	* @param pGCPtrEff Where to return the effective address.
8633	* @param puInfo Extra info: 32-bit displacement (bits 31:0) and
8634	* SIB byte (bits 39:32).
8635	*/
8636	VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset, PRTGCPTR pGCPtrEff, uint64_t *puInfo) RT_NOEXCEPT
8637	{
8638	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8639	# define SET_SS_DEF() \
8640	do \
8641	{ \
8642	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8643	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8644	} while (0)
8645
8646	uint64_t uInfo;
8647	if (!IEM_IS_64BIT_CODE(pVCpu))
8648	{
8649	/** @todo Check the effective address size crap! */
8650	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8651	{
8652	uint16_t u16EffAddr;
8653
8654	/* Handle the disp16 form with no registers first. */
8655	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8656	{
8657	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8658	uInfo = u16EffAddr;
8659	}
8660	else
8661	{
8662	/* Get the displacment. */
8663	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8664	{
8665	case 0: u16EffAddr = 0; break;
8666	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8667	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8668	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8669	}
8670	uInfo = u16EffAddr;
8671
8672	/* Add the base and index registers to the disp. */
8673	switch (bRm & X86_MODRM_RM_MASK)
8674	{
8675	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8676	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8677	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8678	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8679	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8680	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8681	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8682	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8683	}
8684	}
8685
8686	*pGCPtrEff = u16EffAddr;
8687	}
8688	else
8689	{
8690	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8691	uint32_t u32EffAddr;
8692
8693	/* Handle the disp32 form with no registers first. */
8694	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8695	{
8696	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8697	uInfo = u32EffAddr;
8698	}
8699	else
8700	{
8701	/* Get the register (or SIB) value. */
8702	uInfo = 0;
8703	switch ((bRm & X86_MODRM_RM_MASK))
8704	{
8705	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8706	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8707	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8708	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8709	case 4: /* SIB */
8710	{
8711	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8712	uInfo = (uint64_t)bSib << 32;
8713
8714	/* Get the index and scale it. */
8715	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8716	{
8717	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8718	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8719	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8720	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8721	case 4: u32EffAddr = 0; /none / break;
8722	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8723	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8724	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8725	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8726	}
8727	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8728
8729	/* add base */
8730	switch (bSib & X86_SIB_BASE_MASK)
8731	{
8732	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8733	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8734	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8735	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8736	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
8737	case 5:
8738	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8739	{
8740	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8741	SET_SS_DEF();
8742	}
8743	else
8744	{
8745	uint32_t u32Disp;
8746	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8747	u32EffAddr += u32Disp;
8748	uInfo \|= u32Disp;
8749	}
8750	break;
8751	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8752	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8753	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8754	}
8755	break;
8756	}
8757	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8758	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8759	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8760	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8761	}
8762
8763	/* Get and add the displacement. */
8764	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8765	{
8766	case 0:
8767	break;
8768	case 1:
8769	{
8770	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8771	u32EffAddr += i8Disp;
8772	uInfo \|= (uint32_t)(int32_t)i8Disp;
8773	break;
8774	}
8775	case 2:
8776	{
8777	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8778	u32EffAddr += u32Disp;
8779	uInfo \|= (uint32_t)u32Disp;
8780	break;
8781	}
8782	default:
8783	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
8784	}
8785
8786	}
8787	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8788	*pGCPtrEff = u32EffAddr;
8789	}
8790	}
8791	else
8792	{
8793	uint64_t u64EffAddr;
8794
8795	/* Handle the rip+disp32 form with no registers first. */
8796	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8797	{
8798	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8799	uInfo = (uint32_t)u64EffAddr;
8800	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
8801	}
8802	else
8803	{
8804	/* Get the register (or SIB) value. */
8805	uInfo = 0;
8806	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
8807	{
8808	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8809	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8810	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8811	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8812	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8813	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8814	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8815	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8816	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8817	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8818	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8819	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8820	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8821	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8822	/* SIB */
8823	case 4:
8824	case 12:
8825	{
8826	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8827	uInfo = (uint64_t)bSib << 32;
8828
8829	/* Get the index and scale it. */
8830	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
8831	{
8832	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8833	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8834	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8835	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8836	case 4: u64EffAddr = 0; /none / break;
8837	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8838	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8839	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8840	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8841	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8842	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8843	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8844	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8845	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8846	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8847	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8848	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8849	}
8850	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8851
8852	/* add base */
8853	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
8854	{
8855	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8856	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8857	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8858	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8859	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
8860	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8861	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8862	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8863	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8864	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8865	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8866	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8867	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8868	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8869	/* complicated encodings */
8870	case 5:
8871	case 13:
8872	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8873	{
8874	if (!pVCpu->iem.s.uRexB)
8875	{
8876	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8877	SET_SS_DEF();
8878	}
8879	else
8880	u64EffAddr += pVCpu->cpum.GstCtx.r13;
8881	}
8882	else
8883	{
8884	uint32_t u32Disp;
8885	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8886	u64EffAddr += (int32_t)u32Disp;
8887	uInfo \|= u32Disp;
8888	}
8889	break;
8890	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8891	}
8892	break;
8893	}
8894	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8895	}
8896
8897	/* Get and add the displacement. */
8898	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8899	{
8900	case 0:
8901	break;
8902	case 1:
8903	{
8904	int8_t i8Disp;
8905	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8906	u64EffAddr += i8Disp;
8907	uInfo \|= (uint32_t)(int32_t)i8Disp;
8908	break;
8909	}
8910	case 2:
8911	{
8912	uint32_t u32Disp;
8913	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8914	u64EffAddr += (int32_t)u32Disp;
8915	uInfo \|= u32Disp;
8916	break;
8917	}
8918	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
8919	}
8920
8921	}
8922
8923	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8924	*pGCPtrEff = u64EffAddr;
8925	else
8926	{
8927	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8928	*pGCPtrEff = u64EffAddr & UINT32_MAX;
8929	}
8930	}
8931	*puInfo = uInfo;
8932
8933	Log5(("iemOpHlpCalcRmEffAddrEx: EffAddr=%#010RGv uInfo=%RX64\n", *pGCPtrEff, uInfo));
8934	return VINF_SUCCESS;
8935	}
8936
8937	/** @} */
8938
8939
8940	#ifdef LOG_ENABLED
8941	/**
8942	* Logs the current instruction.
8943	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
8944	* @param fSameCtx Set if we have the same context information as the VMM,
8945	* clear if we may have already executed an instruction in
8946	* our debug context. When clear, we assume IEMCPU holds
8947	* valid CPU mode info.
8948	*
8949	* The @a fSameCtx parameter is now misleading and obsolete.
8950	* @param pszFunction The IEM function doing the execution.
8951	*/
8952	static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT
8953	{
8954	# ifdef IN_RING3
8955	if (LogIs2Enabled())
8956	{
8957	char szInstr[256];
8958	uint32_t cbInstr = 0;
8959	if (fSameCtx)
8960	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
8961	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
8962	szInstr, sizeof(szInstr), &cbInstr);
8963	else
8964	{
8965	uint32_t fFlags = 0;
8966	switch (IEM_GET_CPU_MODE(pVCpu))
8967	{
8968	case IEMMODE_64BIT: fFlags \|= DBGF_DISAS_FLAGS_64BIT_MODE; break;
8969	case IEMMODE_32BIT: fFlags \|= DBGF_DISAS_FLAGS_32BIT_MODE; break;
8970	case IEMMODE_16BIT:
8971	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
8972	fFlags \|= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
8973	else
8974	fFlags \|= DBGF_DISAS_FLAGS_16BIT_MODE;
8975	break;
8976	}
8977	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
8978	szInstr, sizeof(szInstr), &cbInstr);
8979	}
8980
8981	PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
8982	Log2(("**** %s fExec=%x\n"
8983	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
8984	" eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
8985	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
8986	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
8987	" %s\n"
8988	, pszFunction, pVCpu->iem.s.fExec,
8989	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
8990	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
8991	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
8992	pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
8993	pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
8994	szInstr));
8995
8996	/* This stuff sucks atm. as it fills the log with MSRs. */
8997	//if (LogIs3Enabled())
8998	// DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
8999	}
9000	else
9001	# endif
9002	LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
9003	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
9004	RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
9005	}
9006	#endif /* LOG_ENABLED */
9007
9008
9009	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9010	/**
9011	* Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
9012	* VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
9013	*
9014	* @returns Modified rcStrict.
9015	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9016	* @param rcStrict The instruction execution status.
9017	*/
9018	static VBOXSTRICTRC iemHandleNestedInstructionBoundaryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
9019	{
9020	Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
9021	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF))
9022	{
9023	/* VMX preemption timer takes priority over NMI-window exits. */
9024	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
9025	{
9026	rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
9027	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
9028	}
9029	/*
9030	* Check remaining intercepts.
9031	*
9032	* NMI-window and Interrupt-window VM-exits.
9033	* Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
9034	* Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
9035	*
9036	* See Intel spec. 26.7.6 "NMI-Window Exiting".
9037	* See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9038	*/
9039	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW \| VMCPU_FF_VMX_INT_WINDOW)
9040	&& !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)
9041	&& !TRPMHasTrap(pVCpu))
9042	{
9043	Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
9044	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
9045	&& CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
9046	{
9047	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
9048	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
9049	}
9050	else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
9051	&& CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
9052	{
9053	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
9054	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
9055	}
9056	}
9057	}
9058	/* TPR-below threshold/APIC write has the highest priority. */
9059	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
9060	{
9061	rcStrict = iemVmxApicWriteEmulation(pVCpu);
9062	Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));
9063	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
9064	}
9065	/* MTF takes priority over VMX-preemption timer. */
9066	else
9067	{
9068	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
9069	Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));
9070	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
9071	}
9072	return rcStrict;
9073	}
9074	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9075
9076
9077	/**
9078	* The actual code execution bits of IEMExecOne, IEMExecOneEx, and
9079	* IEMExecOneWithPrefetchedByPC.
9080	*
9081	* Similar code is found in IEMExecLots.
9082	*
9083	* @return Strict VBox status code.
9084	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9085	* @param fExecuteInhibit If set, execute the instruction following CLI,
9086	* POP SS and MOV SS,GR.
9087	* @param pszFunction The calling function name.
9088	*/
9089	DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
9090	{
9091	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9092	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9093	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9094	RT_NOREF_PV(pszFunction);
9095
9096	#ifdef IEM_WITH_SETJMP
9097	VBOXSTRICTRC rcStrict;
9098	IEM_TRY_SETJMP(pVCpu, rcStrict)
9099	{
9100	uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
9101	rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);
9102	}
9103	IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
9104	{
9105	pVCpu->iem.s.cLongJumps++;
9106	}
9107	IEM_CATCH_LONGJMP_END(pVCpu);
9108	#else
9109	uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
9110	VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);
9111	#endif
9112	if (rcStrict == VINF_SUCCESS)
9113	pVCpu->iem.s.cInstructions++;
9114	if (pVCpu->iem.s.cActiveMappings > 0)
9115	{
9116	Assert(rcStrict != VINF_SUCCESS);
9117	iemMemRollback(pVCpu);
9118	}
9119	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9120	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9121	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9122
9123	//#ifdef DEBUG
9124	// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
9125	//#endif
9126
9127	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9128	/*
9129	* Perform any VMX nested-guest instruction boundary actions.
9130	*
9131	* If any of these causes a VM-exit, we must skip executing the next
9132	* instruction (would run into stale page tables). A VM-exit makes sure
9133	* there is no interrupt-inhibition, so that should ensure we don't go
9134	* to try execute the next instruction. Clearing fExecuteInhibit is
9135	* problematic because of the setjmp/longjmp clobbering above.
9136	*/
9137	if ( !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9138	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW)
9139	\|\| rcStrict != VINF_SUCCESS)
9140	{ /* likely */ }
9141	else
9142	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9143	#endif
9144
9145	/* Execute the next instruction as well if a cli, pop ss or
9146	mov ss, Gr has just completed successfully. */
9147	if ( fExecuteInhibit
9148	&& rcStrict == VINF_SUCCESS
9149	&& CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
9150	{
9151	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fExec & (IEM_F_BYPASS_HANDLERS \| IEM_F_X86_DISREGARD_LOCK));
9152	if (rcStrict == VINF_SUCCESS)
9153	{
9154	#ifdef LOG_ENABLED
9155	iemLogCurInstr(pVCpu, false, pszFunction);
9156	#endif
9157	#ifdef IEM_WITH_SETJMP
9158	IEM_TRY_SETJMP_AGAIN(pVCpu, rcStrict)
9159	{
9160	uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
9161	rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);
9162	}
9163	IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
9164	{
9165	pVCpu->iem.s.cLongJumps++;
9166	}
9167	IEM_CATCH_LONGJMP_END(pVCpu);
9168	#else
9169	IEM_OPCODE_GET_FIRST_U8(&b);
9170	rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);
9171	#endif
9172	if (rcStrict == VINF_SUCCESS)
9173	{
9174	pVCpu->iem.s.cInstructions++;
9175	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9176	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9177	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW))
9178	{ /* likely */ }
9179	else
9180	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9181	#endif
9182	}
9183	if (pVCpu->iem.s.cActiveMappings > 0)
9184	{
9185	Assert(rcStrict != VINF_SUCCESS);
9186	iemMemRollback(pVCpu);
9187	}
9188	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9189	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9190	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9191	}
9192	else if (pVCpu->iem.s.cActiveMappings > 0)
9193	iemMemRollback(pVCpu);
9194	/** @todo drop this after we bake this change into RIP advancing. */
9195	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); /* hope this is correct for all exceptional cases... */
9196	}
9197
9198	/*
9199	* Return value fiddling, statistics and sanity assertions.
9200	*/
9201	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9202
9203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9205	return rcStrict;
9206	}
9207
9208
9209	/**
9210	* Execute one instruction.
9211	*
9212	* @return Strict VBox status code.
9213	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9214	*/
9215	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
9216	{
9217	AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
9218	#ifdef LOG_ENABLED
9219	iemLogCurInstr(pVCpu, true, "IEMExecOne");
9220	#endif
9221
9222	/*
9223	* Do the decoding and emulation.
9224	*/
9225	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /fExecOpts/);
9226	if (rcStrict == VINF_SUCCESS)
9227	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
9228	else if (pVCpu->iem.s.cActiveMappings > 0)
9229	iemMemRollback(pVCpu);
9230
9231	if (rcStrict != VINF_SUCCESS)
9232	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9233	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9234	return rcStrict;
9235	}
9236
9237
9238	VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, uint32_t *pcbWritten)
9239	{
9240	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9241	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /fExecOpts/);
9242	if (rcStrict == VINF_SUCCESS)
9243	{
9244	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
9245	if (pcbWritten)
9246	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9247	}
9248	else if (pVCpu->iem.s.cActiveMappings > 0)
9249	iemMemRollback(pVCpu);
9250
9251	return rcStrict;
9252	}
9253
9254
9255	VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC,
9256	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9257	{
9258	VBOXSTRICTRC rcStrict;
9259	if ( cbOpcodeBytes
9260	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9261	{
9262	iemInitDecoder(pVCpu, 0 /fExecOpts/);
9263	#ifdef IEM_WITH_CODE_TLB
9264	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9265	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9266	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9267	pVCpu->iem.s.offCurInstrStart = 0;
9268	pVCpu->iem.s.offInstrNextByte = 0;
9269	pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
9270	#else
9271	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9272	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9273	#endif
9274	rcStrict = VINF_SUCCESS;
9275	}
9276	else
9277	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /fExecOpts/);
9278	if (rcStrict == VINF_SUCCESS)
9279	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
9280	else if (pVCpu->iem.s.cActiveMappings > 0)
9281	iemMemRollback(pVCpu);
9282
9283	return rcStrict;
9284	}
9285
9286
9287	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, uint32_t *pcbWritten)
9288	{
9289	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9290	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, IEM_F_BYPASS_HANDLERS);
9291	if (rcStrict == VINF_SUCCESS)
9292	{
9293	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
9294	if (pcbWritten)
9295	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9296	}
9297	else if (pVCpu->iem.s.cActiveMappings > 0)
9298	iemMemRollback(pVCpu);
9299
9300	return rcStrict;
9301	}
9302
9303
9304	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC,
9305	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9306	{
9307	VBOXSTRICTRC rcStrict;
9308	if ( cbOpcodeBytes
9309	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9310	{
9311	iemInitDecoder(pVCpu, IEM_F_BYPASS_HANDLERS);
9312	#ifdef IEM_WITH_CODE_TLB
9313	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9314	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9315	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9316	pVCpu->iem.s.offCurInstrStart = 0;
9317	pVCpu->iem.s.offInstrNextByte = 0;
9318	pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
9319	#else
9320	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9321	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9322	#endif
9323	rcStrict = VINF_SUCCESS;
9324	}
9325	else
9326	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, IEM_F_BYPASS_HANDLERS);
9327	if (rcStrict == VINF_SUCCESS)
9328	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
9329	else if (pVCpu->iem.s.cActiveMappings > 0)
9330	iemMemRollback(pVCpu);
9331
9332	return rcStrict;
9333	}
9334
9335
9336	/**
9337	* For handling split cacheline lock operations when the host has split-lock
9338	* detection enabled.
9339	*
9340	* This will cause the interpreter to disregard the lock prefix and implicit
9341	* locking (xchg).
9342	*
9343	* @returns Strict VBox status code.
9344	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9345	*/
9346	VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
9347	{
9348	/*
9349	* Do the decoding and emulation.
9350	*/
9351	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, IEM_F_X86_DISREGARD_LOCK);
9352	if (rcStrict == VINF_SUCCESS)
9353	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
9354	else if (pVCpu->iem.s.cActiveMappings > 0)
9355	iemMemRollback(pVCpu);
9356
9357	if (rcStrict != VINF_SUCCESS)
9358	LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9359	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9360	return rcStrict;
9361	}
9362
9363
9364	/**
9365	* Code common to IEMExecLots and IEMExecRecompilerThreaded that attempts to
9366	* inject a pending TRPM trap.
9367	*/
9368	VBOXSTRICTRC iemExecInjectPendingTrap(PVMCPUCC pVCpu)
9369	{
9370	Assert(TRPMHasTrap(pVCpu));
9371
9372	if ( !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)
9373	&& !CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
9374	{
9375	/** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
9376	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9377	bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
9378	if (fIntrEnabled)
9379	{
9380	if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
9381	fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9382	else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
9383	fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
9384	else
9385	{
9386	Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
9387	fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
9388	}
9389	}
9390	#else
9391	bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9392	#endif
9393	if (fIntrEnabled)
9394	{
9395	uint8_t u8TrapNo;
9396	TRPMEVENT enmType;
9397	uint32_t uErrCode;
9398	RTGCPTR uCr2;
9399	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /pu8InstLen/, NULL /fIcebp/);
9400	AssertRC(rc2);
9401	Assert(enmType == TRPM_HARDWARE_INT);
9402	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /cbInstr/);
9403
9404	TRPMResetTrap(pVCpu);
9405
9406	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9407	/* Injecting an event may cause a VM-exit. */
9408	if ( rcStrict != VINF_SUCCESS
9409	&& rcStrict != VINF_IEM_RAISED_XCPT)
9410	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
9411	#else
9412	NOREF(rcStrict);
9413	#endif
9414	}
9415	}
9416
9417	return VINF_SUCCESS;
9418	}
9419
9420
9421	VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
9422	{
9423	uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
9424	AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
9425	Assert(cMaxInstructions > 0);
9426
9427	/*
9428	* See if there is an interrupt pending in TRPM, inject it if we can.
9429	*/
9430	/** @todo What if we are injecting an exception and not an interrupt? Is that
9431	* possible here? For now we assert it is indeed only an interrupt. */
9432	if (!TRPMHasTrap(pVCpu))
9433	{ /* likely */ }
9434	else
9435	{
9436	VBOXSTRICTRC rcStrict = iemExecInjectPendingTrap(pVCpu);
9437	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9438	{ /likely / }
9439	else
9440	return rcStrict;
9441	}
9442
9443	/*
9444	* Initial decoder init w/ prefetch, then setup setjmp.
9445	*/
9446	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /fExecOpts/);
9447	if (rcStrict == VINF_SUCCESS)
9448	{
9449	#ifdef IEM_WITH_SETJMP
9450	pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf? */
9451	IEM_TRY_SETJMP(pVCpu, rcStrict)
9452	#endif
9453	{
9454	/*
9455	* The run loop. We limit ourselves to 4096 instructions right now.
9456	*/
9457	uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
9458	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
9459	for (;;)
9460	{
9461	/*
9462	* Log the state.
9463	*/
9464	#ifdef LOG_ENABLED
9465	iemLogCurInstr(pVCpu, true, "IEMExecLots");
9466	#endif
9467
9468	/*
9469	* Do the decoding and emulation.
9470	*/
9471	uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
9472	rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);
9473	#ifdef VBOX_STRICT
9474	CPUMAssertGuestRFlagsCookie(pVM, pVCpu);
9475	#endif
9476	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9477	{
9478	Assert(pVCpu->iem.s.cActiveMappings == 0);
9479	pVCpu->iem.s.cInstructions++;
9480
9481	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9482	/* Perform any VMX nested-guest instruction boundary actions. */
9483	uint64_t fCpu = pVCpu->fLocalForcedActions;
9484	if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9485	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW)))
9486	{ /* likely */ }
9487	else
9488	{
9489	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9490	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9491	fCpu = pVCpu->fLocalForcedActions;
9492	else
9493	{
9494	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9495	break;
9496	}
9497	}
9498	#endif
9499	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
9500	{
9501	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
9502	uint64_t fCpu = pVCpu->fLocalForcedActions;
9503	#endif
9504	fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
9505	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
9506	\| VMCPU_FF_TLB_FLUSH
9507	\| VMCPU_FF_UNHALT );
9508
9509	if (RT_LIKELY( ( !fCpu
9510	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
9511	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
9512	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
9513	{
9514	if (--cMaxInstructionsGccStupidity > 0)
9515	{
9516	/* Poll timers every now an then according to the caller's specs. */
9517	if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
9518	\|\| !TMTimerPollBool(pVM, pVCpu))
9519	{
9520	Assert(pVCpu->iem.s.cActiveMappings == 0);
9521	iemReInitDecoder(pVCpu);
9522	continue;
9523	}
9524	}
9525	}
9526	}
9527	Assert(pVCpu->iem.s.cActiveMappings == 0);
9528	}
9529	else if (pVCpu->iem.s.cActiveMappings > 0)
9530	iemMemRollback(pVCpu);
9531	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9532	break;
9533	}
9534	}
9535	#ifdef IEM_WITH_SETJMP
9536	IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
9537	{
9538	if (pVCpu->iem.s.cActiveMappings > 0)
9539	iemMemRollback(pVCpu);
9540	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9541	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9542	# endif
9543	pVCpu->iem.s.cLongJumps++;
9544	}
9545	IEM_CATCH_LONGJMP_END(pVCpu);
9546	#endif
9547
9548	/*
9549	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
9550	*/
9551	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9552	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9553	}
9554	else
9555	{
9556	if (pVCpu->iem.s.cActiveMappings > 0)
9557	iemMemRollback(pVCpu);
9558
9559	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9560	/*
9561	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
9562	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
9563	*/
9564	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9565	#endif
9566	}
9567
9568	/*
9569	* Maybe re-enter raw-mode and log.
9570	*/
9571	if (rcStrict != VINF_SUCCESS)
9572	LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9573	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9574	if (pcInstructions)
9575	*pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
9576	return rcStrict;
9577	}
9578
9579
9580	/**
9581	* Interface used by EMExecuteExec, does exit statistics and limits.
9582	*
9583	* @returns Strict VBox status code.
9584	* @param pVCpu The cross context virtual CPU structure.
9585	* @param fWillExit To be defined.
9586	* @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
9587	* @param cMaxInstructions Maximum number of instructions to execute.
9588	* @param cMaxInstructionsWithoutExits
9589	* The max number of instructions without exits.
9590	* @param pStats Where to return statistics.
9591	*/
9592	VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
9593	uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
9594	{
9595	NOREF(fWillExit); /** @todo define flexible exit crits */
9596
9597	/*
9598	* Initialize return stats.
9599	*/
9600	pStats->cInstructions = 0;
9601	pStats->cExits = 0;
9602	pStats->cMaxExitDistance = 0;
9603	pStats->cReserved = 0;
9604
9605	/*
9606	* Initial decoder init w/ prefetch, then setup setjmp.
9607	*/
9608	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, 0 /fExecOpts/);
9609	if (rcStrict == VINF_SUCCESS)
9610	{
9611	#ifdef IEM_WITH_SETJMP
9612	pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf?!? */
9613	IEM_TRY_SETJMP(pVCpu, rcStrict)
9614	#endif
9615	{
9616	#ifdef IN_RING0
9617	bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() \|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
9618	#endif
9619	uint32_t cInstructionSinceLastExit = 0;
9620
9621	/*
9622	* The run loop. We limit ourselves to 4096 instructions right now.
9623	*/
9624	PVM pVM = pVCpu->CTX_SUFF(pVM);
9625	for (;;)
9626	{
9627	/*
9628	* Log the state.
9629	*/
9630	#ifdef LOG_ENABLED
9631	iemLogCurInstr(pVCpu, true, "IEMExecForExits");
9632	#endif
9633
9634	/*
9635	* Do the decoding and emulation.
9636	*/
9637	uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
9638
9639	uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
9640	rcStrict = FNIEMOP_CALL(g_apfnIemInterpretOnlyOneByteMap[b]);
9641
9642	if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
9643	&& cInstructionSinceLastExit > 0 /* don't count the first */ )
9644	{
9645	pStats->cExits += 1;
9646	if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
9647	pStats->cMaxExitDistance = cInstructionSinceLastExit;
9648	cInstructionSinceLastExit = 0;
9649	}
9650
9651	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9652	{
9653	Assert(pVCpu->iem.s.cActiveMappings == 0);
9654	pVCpu->iem.s.cInstructions++;
9655	pStats->cInstructions++;
9656	cInstructionSinceLastExit++;
9657
9658	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9659	/* Perform any VMX nested-guest instruction boundary actions. */
9660	uint64_t fCpu = pVCpu->fLocalForcedActions;
9661	if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9662	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW)))
9663	{ /* likely */ }
9664	else
9665	{
9666	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9667	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9668	fCpu = pVCpu->fLocalForcedActions;
9669	else
9670	{
9671	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9672	break;
9673	}
9674	}
9675	#endif
9676	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
9677	{
9678	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
9679	uint64_t fCpu = pVCpu->fLocalForcedActions;
9680	#endif
9681	fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
9682	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
9683	\| VMCPU_FF_TLB_FLUSH
9684	\| VMCPU_FF_UNHALT );
9685	if (RT_LIKELY( ( ( !fCpu
9686	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
9687	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
9688	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
9689	\|\| pStats->cInstructions < cMinInstructions))
9690	{
9691	if (pStats->cInstructions < cMaxInstructions)
9692	{
9693	if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
9694	{
9695	#ifdef IN_RING0
9696	if ( !fCheckPreemptionPending
9697	\|\| !RTThreadPreemptIsPending(NIL_RTTHREAD))
9698	#endif
9699	{
9700	Assert(pVCpu->iem.s.cActiveMappings == 0);
9701	iemReInitDecoder(pVCpu);
9702	continue;
9703	}
9704	#ifdef IN_RING0
9705	rcStrict = VINF_EM_RAW_INTERRUPT;
9706	break;
9707	#endif
9708	}
9709	}
9710	}
9711	Assert(!(fCpu & VMCPU_FF_IEM));
9712	}
9713	Assert(pVCpu->iem.s.cActiveMappings == 0);
9714	}
9715	else if (pVCpu->iem.s.cActiveMappings > 0)
9716	iemMemRollback(pVCpu);
9717	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9718	break;
9719	}
9720	}
9721	#ifdef IEM_WITH_SETJMP
9722	IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
9723	{
9724	if (pVCpu->iem.s.cActiveMappings > 0)
9725	iemMemRollback(pVCpu);
9726	pVCpu->iem.s.cLongJumps++;
9727	}
9728	IEM_CATCH_LONGJMP_END(pVCpu);
9729	#endif
9730
9731	/*
9732	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
9733	*/
9734	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9735	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9736	}
9737	else
9738	{
9739	if (pVCpu->iem.s.cActiveMappings > 0)
9740	iemMemRollback(pVCpu);
9741
9742	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9743	/*
9744	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
9745	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
9746	*/
9747	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9748	#endif
9749	}
9750
9751	/*
9752	* Maybe re-enter raw-mode and log.
9753	*/
9754	if (rcStrict != VINF_SUCCESS)
9755	LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
9756	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
9757	pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
9758	return rcStrict;
9759	}
9760
9761
9762	/**
9763	* Injects a trap, fault, abort, software interrupt or external interrupt.
9764	*
9765	* The parameter list matches TRPMQueryTrapAll pretty closely.
9766	*
9767	* @returns Strict VBox status code.
9768	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9769	* @param u8TrapNo The trap number.
9770	* @param enmType What type is it (trap/fault/abort), software
9771	* interrupt or hardware interrupt.
9772	* @param uErrCode The error code if applicable.
9773	* @param uCr2 The CR2 value if applicable.
9774	* @param cbInstr The instruction length (only relevant for
9775	* software interrupts).
9776	*/
9777	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
9778	uint8_t cbInstr)
9779	{
9780	iemInitDecoder(pVCpu, 0 /fExecOpts/); /** @todo wrong init function! */
9781	#ifdef DBGFTRACE_ENABLED
9782	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
9783	u8TrapNo, enmType, uErrCode, uCr2);
9784	#endif
9785
9786	uint32_t fFlags;
9787	switch (enmType)
9788	{
9789	case TRPM_HARDWARE_INT:
9790	Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
9791	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
9792	uErrCode = uCr2 = 0;
9793	break;
9794
9795	case TRPM_SOFTWARE_INT:
9796	Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
9797	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
9798	uErrCode = uCr2 = 0;
9799	break;
9800
9801	case TRPM_TRAP:
9802	case TRPM_NMI: /** @todo Distinguish NMI from exception 2. */
9803	Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
9804	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
9805	if (u8TrapNo == X86_XCPT_PF)
9806	fFlags \|= IEM_XCPT_FLAGS_CR2;
9807	switch (u8TrapNo)
9808	{
9809	case X86_XCPT_DF:
9810	case X86_XCPT_TS:
9811	case X86_XCPT_NP:
9812	case X86_XCPT_SS:
9813	case X86_XCPT_PF:
9814	case X86_XCPT_AC:
9815	case X86_XCPT_GP:
9816	fFlags \|= IEM_XCPT_FLAGS_ERR;
9817	break;
9818	}
9819	break;
9820
9821	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9822	}
9823
9824	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
9825
9826	if (pVCpu->iem.s.cActiveMappings > 0)
9827	iemMemRollback(pVCpu);
9828
9829	return rcStrict;
9830	}
9831
9832
9833	/**
9834	* Injects the active TRPM event.
9835	*
9836	* @returns Strict VBox status code.
9837	* @param pVCpu The cross context virtual CPU structure.
9838	*/
9839	VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
9840	{
9841	#ifndef IEM_IMPLEMENTS_TASKSWITCH
9842	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
9843	#else
9844	uint8_t u8TrapNo;
9845	TRPMEVENT enmType;
9846	uint32_t uErrCode;
9847	RTGCUINTPTR uCr2;
9848	uint8_t cbInstr;
9849	int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
9850	if (RT_FAILURE(rc))
9851	return rc;
9852
9853	/** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
9854	* ICEBP \#DB injection as a special case. */
9855	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
9856	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
9857	if (rcStrict == VINF_SVM_VMEXIT)
9858	rcStrict = VINF_SUCCESS;
9859	#endif
9860	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9861	if (rcStrict == VINF_VMX_VMEXIT)
9862	rcStrict = VINF_SUCCESS;
9863	#endif
9864	/** @todo Are there any other codes that imply the event was successfully
9865	* delivered to the guest? See @bugref{6607}. */
9866	if ( rcStrict == VINF_SUCCESS
9867	\|\| rcStrict == VINF_IEM_RAISED_XCPT)
9868	TRPMResetTrap(pVCpu);
9869
9870	return rcStrict;
9871	#endif
9872	}
9873
9874
9875	VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
9876	{
9877	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
9878	return VERR_NOT_IMPLEMENTED;
9879	}
9880
9881
9882	VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
9883	{
9884	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
9885	return VERR_NOT_IMPLEMENTED;
9886	}
9887
9888
9889	/**
9890	* Interface for HM and EM for executing string I/O OUT (write) instructions.
9891	*
9892	* This API ASSUMES that the caller has already verified that the guest code is
9893	* allowed to access the I/O port. (The I/O port is in the DX register in the
9894	* guest state.)
9895	*
9896	* @returns Strict VBox status code.
9897	* @param pVCpu The cross context virtual CPU structure.
9898	* @param cbValue The size of the I/O port access (1, 2, or 4).
9899	* @param enmAddrMode The addressing mode.
9900	* @param fRepPrefix Indicates whether a repeat prefix is used
9901	* (doesn't matter which for this instruction).
9902	* @param cbInstr The instruction length in bytes.
9903	* @param iEffSeg The effective segment address.
9904	* @param fIoChecked Whether the access to the I/O port has been
9905	* checked or not. It's typically checked in the
9906	* HM scenario.
9907	*/
9908	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
9909	bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
9910	{
9911	AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
9912	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
9913
9914	/*
9915	* State init.
9916	*/
9917	iemInitExec(pVCpu, 0 /fExecOpts/);
9918
9919	/*
9920	* Switch orgy for getting to the right handler.
9921	*/
9922	VBOXSTRICTRC rcStrict;
9923	if (fRepPrefix)
9924	{
9925	switch (enmAddrMode)
9926	{
9927	case IEMMODE_16BIT:
9928	switch (cbValue)
9929	{
9930	case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9931	case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9932	case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9933	default:
9934	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
9935	}
9936	break;
9937
9938	case IEMMODE_32BIT:
9939	switch (cbValue)
9940	{
9941	case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9942	case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9943	case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9944	default:
9945	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
9946	}
9947	break;
9948
9949	case IEMMODE_64BIT:
9950	switch (cbValue)
9951	{
9952	case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9953	case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9954	case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9955	default:
9956	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
9957	}
9958	break;
9959
9960	default:
9961	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
9962	}
9963	}
9964	else
9965	{
9966	switch (enmAddrMode)
9967	{
9968	case IEMMODE_16BIT:
9969	switch (cbValue)
9970	{
9971	case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9972	case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9973	case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9974	default:
9975	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
9976	}
9977	break;
9978
9979	case IEMMODE_32BIT:
9980	switch (cbValue)
9981	{
9982	case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9983	case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9984	case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9985	default:
9986	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
9987	}
9988	break;
9989
9990	case IEMMODE_64BIT:
9991	switch (cbValue)
9992	{
9993	case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9994	case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9995	case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
9996	default:
9997	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
9998	}
9999	break;
10000
10001	default:
10002	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10003	}
10004	}
10005
10006	if (pVCpu->iem.s.cActiveMappings)
10007	iemMemRollback(pVCpu);
10008
10009	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10010	}
10011
10012
10013	/**
10014	* Interface for HM and EM for executing string I/O IN (read) instructions.
10015	*
10016	* This API ASSUMES that the caller has already verified that the guest code is
10017	* allowed to access the I/O port. (The I/O port is in the DX register in the
10018	* guest state.)
10019	*
10020	* @returns Strict VBox status code.
10021	* @param pVCpu The cross context virtual CPU structure.
10022	* @param cbValue The size of the I/O port access (1, 2, or 4).
10023	* @param enmAddrMode The addressing mode.
10024	* @param fRepPrefix Indicates whether a repeat prefix is used
10025	* (doesn't matter which for this instruction).
10026	* @param cbInstr The instruction length in bytes.
10027	* @param fIoChecked Whether the access to the I/O port has been
10028	* checked or not. It's typically checked in the
10029	* HM scenario.
10030	*/
10031	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10032	bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
10033	{
10034	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10035
10036	/*
10037	* State init.
10038	*/
10039	iemInitExec(pVCpu, 0 /fExecOpts/);
10040
10041	/*
10042	* Switch orgy for getting to the right handler.
10043	*/
10044	VBOXSTRICTRC rcStrict;
10045	if (fRepPrefix)
10046	{
10047	switch (enmAddrMode)
10048	{
10049	case IEMMODE_16BIT:
10050	switch (cbValue)
10051	{
10052	case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10053	case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10054	case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10055	default:
10056	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10057	}
10058	break;
10059
10060	case IEMMODE_32BIT:
10061	switch (cbValue)
10062	{
10063	case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10064	case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10065	case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10066	default:
10067	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10068	}
10069	break;
10070
10071	case IEMMODE_64BIT:
10072	switch (cbValue)
10073	{
10074	case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10075	case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10076	case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10077	default:
10078	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10079	}
10080	break;
10081
10082	default:
10083	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10084	}
10085	}
10086	else
10087	{
10088	switch (enmAddrMode)
10089	{
10090	case IEMMODE_16BIT:
10091	switch (cbValue)
10092	{
10093	case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10094	case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10095	case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10096	default:
10097	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10098	}
10099	break;
10100
10101	case IEMMODE_32BIT:
10102	switch (cbValue)
10103	{
10104	case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10105	case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10106	case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10107	default:
10108	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10109	}
10110	break;
10111
10112	case IEMMODE_64BIT:
10113	switch (cbValue)
10114	{
10115	case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10116	case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10117	case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10118	default:
10119	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10120	}
10121	break;
10122
10123	default:
10124	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10125	}
10126	}
10127
10128	if ( pVCpu->iem.s.cActiveMappings == 0
10129	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
10130	{ /* likely */ }
10131	else
10132	{
10133	AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
10134	iemMemRollback(pVCpu);
10135	}
10136	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10137	}
10138
10139
10140	/**
10141	* Interface for rawmode to write execute an OUT instruction.
10142	*
10143	* @returns Strict VBox status code.
10144	* @param pVCpu The cross context virtual CPU structure.
10145	* @param cbInstr The instruction length in bytes.
10146	* @param u16Port The port to read.
10147	* @param fImm Whether the port is specified using an immediate operand or
10148	* using the implicit DX register.
10149	* @param cbReg The register size.
10150	*
10151	* @remarks In ring-0 not all of the state needs to be synced in.
10152	*/
10153	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10154	{
10155	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10156	Assert(cbReg <= 4 && cbReg != 3);
10157
10158	iemInitExec(pVCpu, 0 /fExecOpts/);
10159	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, cbReg,
10160	((uint8_t)fImm << 7) \| 0xf /** @todo never worked with intercepts */);
10161	Assert(!pVCpu->iem.s.cActiveMappings);
10162	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10163	}
10164
10165
10166	/**
10167	* Interface for rawmode to write execute an IN instruction.
10168	*
10169	* @returns Strict VBox status code.
10170	* @param pVCpu The cross context virtual CPU structure.
10171	* @param cbInstr The instruction length in bytes.
10172	* @param u16Port The port to read.
10173	* @param fImm Whether the port is specified using an immediate operand or
10174	* using the implicit DX.
10175	* @param cbReg The register size.
10176	*/
10177	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10178	{
10179	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10180	Assert(cbReg <= 4 && cbReg != 3);
10181
10182	iemInitExec(pVCpu, 0 /fExecOpts/);
10183	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, cbReg,
10184	((uint8_t)fImm << 7) \| 0xf /** @todo never worked with intercepts */);
10185	Assert(!pVCpu->iem.s.cActiveMappings);
10186	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10187	}
10188
10189
10190	/**
10191	* Interface for HM and EM to write to a CRx register.
10192	*
10193	* @returns Strict VBox status code.
10194	* @param pVCpu The cross context virtual CPU structure.
10195	* @param cbInstr The instruction length in bytes.
10196	* @param iCrReg The control register number (destination).
10197	* @param iGReg The general purpose register number (source).
10198	*
10199	* @remarks In ring-0 not all of the state needs to be synced in.
10200	*/
10201	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
10202	{
10203	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10204	Assert(iCrReg < 16);
10205	Assert(iGReg < 16);
10206
10207	iemInitExec(pVCpu, 0 /fExecOpts/);
10208	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
10209	Assert(!pVCpu->iem.s.cActiveMappings);
10210	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10211	}
10212
10213
10214	/**
10215	* Interface for HM and EM to read from a CRx register.
10216	*
10217	* @returns Strict VBox status code.
10218	* @param pVCpu The cross context virtual CPU structure.
10219	* @param cbInstr The instruction length in bytes.
10220	* @param iGReg The general purpose register number (destination).
10221	* @param iCrReg The control register number (source).
10222	*
10223	* @remarks In ring-0 not all of the state needs to be synced in.
10224	*/
10225	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10226	{
10227	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10228	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
10229	\| CPUMCTX_EXTRN_APIC_TPR);
10230	Assert(iCrReg < 16);
10231	Assert(iGReg < 16);
10232
10233	iemInitExec(pVCpu, 0 /fExecOpts/);
10234	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
10235	Assert(!pVCpu->iem.s.cActiveMappings);
10236	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10237	}
10238
10239
10240	/**
10241	* Interface for HM and EM to write to a DRx register.
10242	*
10243	* @returns Strict VBox status code.
10244	* @param pVCpu The cross context virtual CPU structure.
10245	* @param cbInstr The instruction length in bytes.
10246	* @param iDrReg The debug register number (destination).
10247	* @param iGReg The general purpose register number (source).
10248	*
10249	* @remarks In ring-0 not all of the state needs to be synced in.
10250	*/
10251	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovDRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iDrReg, uint8_t iGReg)
10252	{
10253	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10254	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_DR7);
10255	Assert(iDrReg < 8);
10256	Assert(iGReg < 16);
10257
10258	iemInitExec(pVCpu, 0 /fExecOpts/);
10259	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Dd_Rd, iDrReg, iGReg);
10260	Assert(!pVCpu->iem.s.cActiveMappings);
10261	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10262	}
10263
10264
10265	/**
10266	* Interface for HM and EM to read from a DRx register.
10267	*
10268	* @returns Strict VBox status code.
10269	* @param pVCpu The cross context virtual CPU structure.
10270	* @param cbInstr The instruction length in bytes.
10271	* @param iGReg The general purpose register number (destination).
10272	* @param iDrReg The debug register number (source).
10273	*
10274	* @remarks In ring-0 not all of the state needs to be synced in.
10275	*/
10276	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovDRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iDrReg)
10277	{
10278	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10279	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_DR7);
10280	Assert(iDrReg < 8);
10281	Assert(iGReg < 16);
10282
10283	iemInitExec(pVCpu, 0 /fExecOpts/);
10284	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Dd, iGReg, iDrReg);
10285	Assert(!pVCpu->iem.s.cActiveMappings);
10286	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10287	}
10288
10289
10290	/**
10291	* Interface for HM and EM to clear the CR0[TS] bit.
10292	*
10293	* @returns Strict VBox status code.
10294	* @param pVCpu The cross context virtual CPU structure.
10295	* @param cbInstr The instruction length in bytes.
10296	*
10297	* @remarks In ring-0 not all of the state needs to be synced in.
10298	*/
10299	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
10300	{
10301	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10302
10303	iemInitExec(pVCpu, 0 /fExecOpts/);
10304	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
10305	Assert(!pVCpu->iem.s.cActiveMappings);
10306	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10307	}
10308
10309
10310	/**
10311	* Interface for HM and EM to emulate the LMSW instruction (loads CR0).
10312	*
10313	* @returns Strict VBox status code.
10314	* @param pVCpu The cross context virtual CPU structure.
10315	* @param cbInstr The instruction length in bytes.
10316	* @param uValue The value to load into CR0.
10317	* @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
10318	* memory operand. Otherwise pass NIL_RTGCPTR.
10319	*
10320	* @remarks In ring-0 not all of the state needs to be synced in.
10321	*/
10322	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
10323	{
10324	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10325
10326	iemInitExec(pVCpu, 0 /fExecOpts/);
10327	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
10328	Assert(!pVCpu->iem.s.cActiveMappings);
10329	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10330	}
10331
10332
10333	/**
10334	* Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
10335	*
10336	* Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
10337	*
10338	* @returns Strict VBox status code.
10339	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10340	* @param cbInstr The instruction length in bytes.
10341	* @remarks In ring-0 not all of the state needs to be synced in.
10342	* @thread EMT(pVCpu)
10343	*/
10344	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
10345	{
10346	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10347
10348	iemInitExec(pVCpu, 0 /fExecOpts/);
10349	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
10350	Assert(!pVCpu->iem.s.cActiveMappings);
10351	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10352	}
10353
10354
10355	/**
10356	* Interface for HM and EM to emulate the WBINVD instruction.
10357	*
10358	* @returns Strict VBox status code.
10359	* @param pVCpu The cross context virtual CPU structure.
10360	* @param cbInstr The instruction length in bytes.
10361	*
10362	* @remarks In ring-0 not all of the state needs to be synced in.
10363	*/
10364	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10365	{
10366	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10367
10368	iemInitExec(pVCpu, 0 /fExecOpts/);
10369	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
10370	Assert(!pVCpu->iem.s.cActiveMappings);
10371	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10372	}
10373
10374
10375	/**
10376	* Interface for HM and EM to emulate the INVD instruction.
10377	*
10378	* @returns Strict VBox status code.
10379	* @param pVCpu The cross context virtual CPU structure.
10380	* @param cbInstr The instruction length in bytes.
10381	*
10382	* @remarks In ring-0 not all of the state needs to be synced in.
10383	*/
10384	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10385	{
10386	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10387
10388	iemInitExec(pVCpu, 0 /fExecOpts/);
10389	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
10390	Assert(!pVCpu->iem.s.cActiveMappings);
10391	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10392	}
10393
10394
10395	/**
10396	* Interface for HM and EM to emulate the INVLPG instruction.
10397	*
10398	* @returns Strict VBox status code.
10399	* @retval VINF_PGM_SYNC_CR3
10400	*
10401	* @param pVCpu The cross context virtual CPU structure.
10402	* @param cbInstr The instruction length in bytes.
10403	* @param GCPtrPage The effective address of the page to invalidate.
10404	*
10405	* @remarks In ring-0 not all of the state needs to be synced in.
10406	*/
10407	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
10408	{
10409	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10410
10411	iemInitExec(pVCpu, 0 /fExecOpts/);
10412	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
10413	Assert(!pVCpu->iem.s.cActiveMappings);
10414	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10415	}
10416
10417
10418	/**
10419	* Interface for HM and EM to emulate the INVPCID instruction.
10420	*
10421	* @returns Strict VBox status code.
10422	* @retval VINF_PGM_SYNC_CR3
10423	*
10424	* @param pVCpu The cross context virtual CPU structure.
10425	* @param cbInstr The instruction length in bytes.
10426	* @param iEffSeg The effective segment register.
10427	* @param GCPtrDesc The effective address of the INVPCID descriptor.
10428	* @param uType The invalidation type.
10429	*
10430	* @remarks In ring-0 not all of the state needs to be synced in.
10431	*/
10432	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
10433	uint64_t uType)
10434	{
10435	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
10436
10437	iemInitExec(pVCpu, 0 /fExecOpts/);
10438	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
10439	Assert(!pVCpu->iem.s.cActiveMappings);
10440	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10441	}
10442
10443
10444	/**
10445	* Interface for HM and EM to emulate the CPUID instruction.
10446	*
10447	* @returns Strict VBox status code.
10448	*
10449	* @param pVCpu The cross context virtual CPU structure.
10450	* @param cbInstr The instruction length in bytes.
10451	*
10452	* @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
10453	*/
10454	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
10455	{
10456	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10457	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
10458
10459	iemInitExec(pVCpu, 0 /fExecOpts/);
10460	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
10461	Assert(!pVCpu->iem.s.cActiveMappings);
10462	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10463	}
10464
10465
10466	/**
10467	* Interface for HM and EM to emulate the RDPMC instruction.
10468	*
10469	* @returns Strict VBox status code.
10470	*
10471	* @param pVCpu The cross context virtual CPU structure.
10472	* @param cbInstr The instruction length in bytes.
10473	*
10474	* @remarks Not all of the state needs to be synced in.
10475	*/
10476	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
10477	{
10478	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10479	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
10480
10481	iemInitExec(pVCpu, 0 /fExecOpts/);
10482	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
10483	Assert(!pVCpu->iem.s.cActiveMappings);
10484	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10485	}
10486
10487
10488	/**
10489	* Interface for HM and EM to emulate the RDTSC instruction.
10490	*
10491	* @returns Strict VBox status code.
10492	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10493	*
10494	* @param pVCpu The cross context virtual CPU structure.
10495	* @param cbInstr The instruction length in bytes.
10496	*
10497	* @remarks Not all of the state needs to be synced in.
10498	*/
10499	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
10500	{
10501	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10502	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
10503
10504	iemInitExec(pVCpu, 0 /fExecOpts/);
10505	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
10506	Assert(!pVCpu->iem.s.cActiveMappings);
10507	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10508	}
10509
10510
10511	/**
10512	* Interface for HM and EM to emulate the RDTSCP instruction.
10513	*
10514	* @returns Strict VBox status code.
10515	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10516	*
10517	* @param pVCpu The cross context virtual CPU structure.
10518	* @param cbInstr The instruction length in bytes.
10519	*
10520	* @remarks Not all of the state needs to be synced in. Recommended
10521	* to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
10522	*/
10523	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
10524	{
10525	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10526	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
10527
10528	iemInitExec(pVCpu, 0 /fExecOpts/);
10529	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
10530	Assert(!pVCpu->iem.s.cActiveMappings);
10531	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10532	}
10533
10534
10535	/**
10536	* Interface for HM and EM to emulate the RDMSR instruction.
10537	*
10538	* @returns Strict VBox status code.
10539	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10540	*
10541	* @param pVCpu The cross context virtual CPU structure.
10542	* @param cbInstr The instruction length in bytes.
10543	*
10544	* @remarks Not all of the state needs to be synced in. Requires RCX and
10545	* (currently) all MSRs.
10546	*/
10547	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
10548	{
10549	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10550	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_ALL_MSRS);
10551
10552	iemInitExec(pVCpu, 0 /fExecOpts/);
10553	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
10554	Assert(!pVCpu->iem.s.cActiveMappings);
10555	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10556	}
10557
10558
10559	/**
10560	* Interface for HM and EM to emulate the WRMSR instruction.
10561	*
10562	* @returns Strict VBox status code.
10563	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10564	*
10565	* @param pVCpu The cross context virtual CPU structure.
10566	* @param cbInstr The instruction length in bytes.
10567	*
10568	* @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
10569	* and (currently) all MSRs.
10570	*/
10571	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
10572	{
10573	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10574	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
10575	\| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_ALL_MSRS);
10576
10577	iemInitExec(pVCpu, 0 /fExecOpts/);
10578	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
10579	Assert(!pVCpu->iem.s.cActiveMappings);
10580	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10581	}
10582
10583
10584	/**
10585	* Interface for HM and EM to emulate the MONITOR instruction.
10586	*
10587	* @returns Strict VBox status code.
10588	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10589	*
10590	* @param pVCpu The cross context virtual CPU structure.
10591	* @param cbInstr The instruction length in bytes.
10592	*
10593	* @remarks Not all of the state needs to be synced in.
10594	* @remarks ASSUMES the default segment of DS and no segment override prefixes
10595	* are used.
10596	*/
10597	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
10598	{
10599	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10600	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
10601
10602	iemInitExec(pVCpu, 0 /fExecOpts/);
10603	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
10604	Assert(!pVCpu->iem.s.cActiveMappings);
10605	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10606	}
10607
10608
10609	/**
10610	* Interface for HM and EM to emulate the MWAIT instruction.
10611	*
10612	* @returns Strict VBox status code.
10613	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10614	*
10615	* @param pVCpu The cross context virtual CPU structure.
10616	* @param cbInstr The instruction length in bytes.
10617	*
10618	* @remarks Not all of the state needs to be synced in.
10619	*/
10620	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
10621	{
10622	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10623	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX);
10624
10625	iemInitExec(pVCpu, 0 /fExecOpts/);
10626	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
10627	Assert(!pVCpu->iem.s.cActiveMappings);
10628	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10629	}
10630
10631
10632	/**
10633	* Interface for HM and EM to emulate the HLT instruction.
10634	*
10635	* @returns Strict VBox status code.
10636	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
10637	*
10638	* @param pVCpu The cross context virtual CPU structure.
10639	* @param cbInstr The instruction length in bytes.
10640	*
10641	* @remarks Not all of the state needs to be synced in.
10642	*/
10643	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
10644	{
10645	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10646
10647	iemInitExec(pVCpu, 0 /fExecOpts/);
10648	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
10649	Assert(!pVCpu->iem.s.cActiveMappings);
10650	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10651	}
10652
10653
10654	/**
10655	* Checks if IEM is in the process of delivering an event (interrupt or
10656	* exception).
10657	*
10658	* @returns true if we're in the process of raising an interrupt or exception,
10659	* false otherwise.
10660	* @param pVCpu The cross context virtual CPU structure.
10661	* @param puVector Where to store the vector associated with the
10662	* currently delivered event, optional.
10663	* @param pfFlags Where to store th event delivery flags (see
10664	* IEM_XCPT_FLAGS_XXX), optional.
10665	* @param puErr Where to store the error code associated with the
10666	* event, optional.
10667	* @param puCr2 Where to store the CR2 associated with the event,
10668	* optional.
10669	* @remarks The caller should check the flags to determine if the error code and
10670	* CR2 are valid for the event.
10671	*/
10672	VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t puVector, uint32_t pfFlags, uint32_t puErr, uint64_t puCr2)
10673	{
10674	bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
10675	if (fRaisingXcpt)
10676	{
10677	if (puVector)
10678	*puVector = pVCpu->iem.s.uCurXcpt;
10679	if (pfFlags)
10680	*pfFlags = pVCpu->iem.s.fCurXcpt;
10681	if (puErr)
10682	*puErr = pVCpu->iem.s.uCurXcptErr;
10683	if (puCr2)
10684	*puCr2 = pVCpu->iem.s.uCurXcptCr2;
10685	}
10686	return fRaisingXcpt;
10687	}
10688
10689	#ifdef IN_RING3
10690
10691	/**
10692	* Handles the unlikely and probably fatal merge cases.
10693	*
10694	* @returns Merged status code.
10695	* @param rcStrict Current EM status code.
10696	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
10697	* with @a rcStrict.
10698	* @param iMemMap The memory mapping index. For error reporting only.
10699	* @param pVCpu The cross context virtual CPU structure of the calling
10700	* thread, for error reporting only.
10701	*/
10702	DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
10703	unsigned iMemMap, PVMCPUCC pVCpu)
10704	{
10705	if (RT_FAILURE_NP(rcStrict))
10706	return rcStrict;
10707
10708	if (RT_FAILURE_NP(rcStrictCommit))
10709	return rcStrictCommit;
10710
10711	if (rcStrict == rcStrictCommit)
10712	return rcStrictCommit;
10713
10714	AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
10715	VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
10716	pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
10717	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
10718	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
10719	return VERR_IOM_FF_STATUS_IPE;
10720	}
10721
10722
10723	/**
10724	* Helper for IOMR3ProcessForceFlag.
10725	*
10726	* @returns Merged status code.
10727	* @param rcStrict Current EM status code.
10728	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
10729	* with @a rcStrict.
10730	* @param iMemMap The memory mapping index. For error reporting only.
10731	* @param pVCpu The cross context virtual CPU structure of the calling
10732	* thread, for error reporting only.
10733	*/
10734	DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
10735	{
10736	/* Simple. */
10737	if (RT_LIKELY(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RAW_TO_R3))
10738	return rcStrictCommit;
10739
10740	if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
10741	return rcStrict;
10742
10743	/* EM scheduling status codes. */
10744	if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
10745	&& rcStrict <= VINF_EM_LAST))
10746	{
10747	if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
10748	&& rcStrictCommit <= VINF_EM_LAST))
10749	return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
10750	}
10751
10752	/* Unlikely */
10753	return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
10754	}
10755
10756
10757	/**
10758	* Called by force-flag handling code when VMCPU_FF_IEM is set.
10759	*
10760	* @returns Merge between @a rcStrict and what the commit operation returned.
10761	* @param pVM The cross context VM structure.
10762	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10763	* @param rcStrict The status code returned by ring-0 or raw-mode.
10764	*/
10765	VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
10766	{
10767	/*
10768	* Reset the pending commit.
10769	*/
10770	AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess \| pVCpu->iem.s.aMemMappings[1].fAccess \| pVCpu->iem.s.aMemMappings[2].fAccess)
10771	& (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND),
10772	("%#x %#x %#x\n",
10773	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
10774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
10775
10776	/*
10777	* Commit the pending bounce buffers (usually just one).
10778	*/
10779	unsigned cBufs = 0;
10780	unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
10781	while (iMemMap-- > 0)
10782	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND))
10783	{
10784	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
10785	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
10786	Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
10787
10788	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
10789	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
10790	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
10791
10792	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
10793	{
10794	VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
10795	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
10796	pbBuf,
10797	cbFirst,
10798	PGMACCESSORIGIN_IEM);
10799	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
10800	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
10801	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
10802	VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
10803	}
10804
10805	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
10806	{
10807	VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
10808	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
10809	pbBuf + cbFirst,
10810	cbSecond,
10811	PGMACCESSORIGIN_IEM);
10812	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
10813	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
10814	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
10815	VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
10816	}
10817	cBufs++;
10818	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
10819	}
10820
10821	AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
10822	("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
10823	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
10824	pVCpu->iem.s.cActiveMappings = 0;
10825	return rcStrict;
10826	}
10827
10828	#endif /* IN_RING3 */
10829

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 103917

Download in other formats: