IEMAll.cpp@ 96789

Last change on this file since 96789 was 96745, checked in by vboxsync, 2 years ago
VMM/IEM: Some micro optimizations around iemHandleNestedInstructionBoundaryFFs. bugref:10092
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 459.4 KB

Line
1	/* $Id: IEMAll.cpp 96745 2022-09-15 11:35:07Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - All Contexts.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/** @page pg_iem IEM - Interpreted Execution Manager
30	*
31	* The interpreted exeuction manager (IEM) is for executing short guest code
32	* sequences that are causing too many exits / virtualization traps. It will
33	* also be used to interpret single instructions, thus replacing the selective
34	* interpreters in EM and IOM.
35	*
36	* Design goals:
37	* - Relatively small footprint, although we favour speed and correctness
38	* over size.
39	* - Reasonably fast.
40	* - Correctly handle lock prefixed instructions.
41	* - Complete instruction set - eventually.
42	* - Refactorable into a recompiler, maybe.
43	* - Replace EMInterpret*.
44	*
45	* Using the existing disassembler has been considered, however this is thought
46	* to conflict with speed as the disassembler chews things a bit too much while
47	* leaving us with a somewhat complicated state to interpret afterwards.
48	*
49	*
50	* The current code is very much work in progress. You've been warned!
51	*
52	*
53	* @section sec_iem_fpu_instr FPU Instructions
54	*
55	* On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
56	* same or equivalent instructions on the host FPU. To make life easy, we also
57	* let the FPU prioritize the unmasked exceptions for us. This however, only
58	* works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
59	* for FPU exception delivery, because with CR0.NE=0 there is a window where we
60	* can trigger spurious FPU exceptions.
61	*
62	* The guest FPU state is not loaded into the host CPU and kept there till we
63	* leave IEM because the calling conventions have declared an all year open
64	* season on much of the FPU state. For instance an innocent looking call to
65	* memcpy might end up using a whole bunch of XMM or MM registers if the
66	* particular implementation finds it worthwhile.
67	*
68	*
69	* @section sec_iem_logging Logging
70	*
71	* The IEM code uses the \"IEM\" log group for the main logging. The different
72	* logging levels/flags are generally used for the following purposes:
73	* - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
74	* - Flow (LogFlow) : Basic enter/exit IEM state info.
75	* - Level 2 (Log2) : ?
76	* - Level 3 (Log3) : More detailed enter/exit IEM state info.
77	* - Level 4 (Log4) : Decoding mnemonics w/ EIP.
78	* - Level 5 (Log5) : Decoding details.
79	* - Level 6 (Log6) : Enables/disables the lockstep comparison with REM.
80	* - Level 7 (Log7) : iret++ execution logging.
81	* - Level 8 (Log8) : Memory writes.
82	* - Level 9 (Log9) : Memory reads.
83	* - Level 10 (Log10): TLBs.
84	* - Level 11 (Log11): Unmasked FPU exceptions.
85	*/
86
87	/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
88	#ifdef _MSC_VER
89	# pragma warning(disable:4505)
90	#endif
91
92
93	/*********************************************************************************************************************************
94	* Header Files *
95	*********************************************************************************************************************************/
96	#define LOG_GROUP LOG_GROUP_IEM
97	#define VMCPU_INCL_CPUM_GST_CTX
98	#include <VBox/vmm/iem.h>
99	#include <VBox/vmm/cpum.h>
100	#include <VBox/vmm/apic.h>
101	#include <VBox/vmm/pdm.h>
102	#include <VBox/vmm/pgm.h>
103	#include <VBox/vmm/iom.h>
104	#include <VBox/vmm/em.h>
105	#include <VBox/vmm/hm.h>
106	#include <VBox/vmm/nem.h>
107	#include <VBox/vmm/gim.h>
108	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
109	# include <VBox/vmm/em.h>
110	# include <VBox/vmm/hm_svm.h>
111	#endif
112	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
113	# include <VBox/vmm/hmvmxinline.h>
114	#endif
115	#include <VBox/vmm/tm.h>
116	#include <VBox/vmm/dbgf.h>
117	#include <VBox/vmm/dbgftrace.h>
118	#include "IEMInternal.h"
119	#include <VBox/vmm/vmcc.h>
120	#include <VBox/log.h>
121	#include <VBox/err.h>
122	#include <VBox/param.h>
123	#include <VBox/dis.h>
124	#include <VBox/disopcode.h>
125	#include <iprt/asm-math.h>
126	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
127	# include <iprt/asm-amd64-x86.h>
128	#elif defined(RT_ARCH_ARM64) \|\| defined(RT_ARCH_ARM32)
129	# include <iprt/asm-arm.h>
130	#endif
131	#include <iprt/assert.h>
132	#include <iprt/string.h>
133	#include <iprt/x86.h>
134
135	#include "IEMInline.h"
136
137
138	/*********************************************************************************************************************************
139	* Structures and Typedefs *
140	*********************************************************************************************************************************/
141	/**
142	* CPU exception classes.
143	*/
144	typedef enum IEMXCPTCLASS
145	{
146	IEMXCPTCLASS_BENIGN,
147	IEMXCPTCLASS_CONTRIBUTORY,
148	IEMXCPTCLASS_PAGE_FAULT,
149	IEMXCPTCLASS_DOUBLE_FAULT
150	} IEMXCPTCLASS;
151
152
153	/*********************************************************************************************************************************
154	* Global Variables *
155	*********************************************************************************************************************************/
156	#if defined(IEM_LOG_MEMORY_WRITES)
157	/** What IEM just wrote. */
158	uint8_t g_abIemWrote[256];
159	/** How much IEM just wrote. */
160	size_t g_cbIemWrote;
161	#endif
162
163
164	/*********************************************************************************************************************************
165	* Internal Functions *
166	*********************************************************************************************************************************/
167	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
168	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT;
169
170
171	/**
172	* Initializes the decoder state.
173	*
174	* iemReInitDecoder is mostly a copy of this function.
175	*
176	* @param pVCpu The cross context virtual CPU structure of the
177	* calling thread.
178	* @param fBypassHandlers Whether to bypass access handlers.
179	* @param fDisregardLock Whether to disregard the LOCK prefix.
180	*/
181	DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
182	{
183	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
184	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
185	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
186	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
187	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
188	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
189	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
190	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
191	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
192	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
193
194	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
195	IEMMODE enmMode = iemCalcCpuMode(pVCpu);
196	pVCpu->iem.s.enmCpuMode = enmMode;
197	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
198	pVCpu->iem.s.enmEffAddrMode = enmMode;
199	if (enmMode != IEMMODE_64BIT)
200	{
201	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
202	pVCpu->iem.s.enmEffOpSize = enmMode;
203	}
204	else
205	{
206	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
207	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
208	}
209	pVCpu->iem.s.fPrefixes = 0;
210	pVCpu->iem.s.uRexReg = 0;
211	pVCpu->iem.s.uRexB = 0;
212	pVCpu->iem.s.uRexIndex = 0;
213	pVCpu->iem.s.idxPrefix = 0;
214	pVCpu->iem.s.uVex3rdReg = 0;
215	pVCpu->iem.s.uVexLength = 0;
216	pVCpu->iem.s.fEvexStuff = 0;
217	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
218	#ifdef IEM_WITH_CODE_TLB
219	pVCpu->iem.s.pbInstrBuf = NULL;
220	pVCpu->iem.s.offInstrNextByte = 0;
221	pVCpu->iem.s.offCurInstrStart = 0;
222	# ifdef VBOX_STRICT
223	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
224	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
225	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
226	# endif
227	#else
228	pVCpu->iem.s.offOpcode = 0;
229	pVCpu->iem.s.cbOpcode = 0;
230	#endif
231	pVCpu->iem.s.offModRm = 0;
232	pVCpu->iem.s.cActiveMappings = 0;
233	pVCpu->iem.s.iNextMapping = 0;
234	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
235	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
236	pVCpu->iem.s.fDisregardLock = fDisregardLock;
237
238	#ifdef DBGFTRACE_ENABLED
239	switch (enmMode)
240	{
241	case IEMMODE_64BIT:
242	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
243	break;
244	case IEMMODE_32BIT:
245	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
246	break;
247	case IEMMODE_16BIT:
248	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
249	break;
250	}
251	#endif
252	}
253
254
255	/**
256	* Reinitializes the decoder state 2nd+ loop of IEMExecLots.
257	*
258	* This is mostly a copy of iemInitDecoder.
259	*
260	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
261	*/
262	DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
263	{
264	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
265	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
268	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
269	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
270	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
271	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
272	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
273
274	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
275	IEMMODE enmMode = iemCalcCpuMode(pVCpu);
276	pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
277	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
278	pVCpu->iem.s.enmEffAddrMode = enmMode;
279	if (enmMode != IEMMODE_64BIT)
280	{
281	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
282	pVCpu->iem.s.enmEffOpSize = enmMode;
283	}
284	else
285	{
286	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
287	pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
288	}
289	pVCpu->iem.s.fPrefixes = 0;
290	pVCpu->iem.s.uRexReg = 0;
291	pVCpu->iem.s.uRexB = 0;
292	pVCpu->iem.s.uRexIndex = 0;
293	pVCpu->iem.s.idxPrefix = 0;
294	pVCpu->iem.s.uVex3rdReg = 0;
295	pVCpu->iem.s.uVexLength = 0;
296	pVCpu->iem.s.fEvexStuff = 0;
297	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
298	#ifdef IEM_WITH_CODE_TLB
299	if (pVCpu->iem.s.pbInstrBuf)
300	{
301	uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
302	- pVCpu->iem.s.uInstrBufPc;
303	if (off < pVCpu->iem.s.cbInstrBufTotal)
304	{
305	pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
306	pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
307	if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
308	pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
309	else
310	pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
311	}
312	else
313	{
314	pVCpu->iem.s.pbInstrBuf = NULL;
315	pVCpu->iem.s.offInstrNextByte = 0;
316	pVCpu->iem.s.offCurInstrStart = 0;
317	pVCpu->iem.s.cbInstrBuf = 0;
318	pVCpu->iem.s.cbInstrBufTotal = 0;
319	}
320	}
321	else
322	{
323	pVCpu->iem.s.offInstrNextByte = 0;
324	pVCpu->iem.s.offCurInstrStart = 0;
325	pVCpu->iem.s.cbInstrBuf = 0;
326	pVCpu->iem.s.cbInstrBufTotal = 0;
327	}
328	#else
329	pVCpu->iem.s.cbOpcode = 0;
330	pVCpu->iem.s.offOpcode = 0;
331	#endif
332	pVCpu->iem.s.offModRm = 0;
333	Assert(pVCpu->iem.s.cActiveMappings == 0);
334	pVCpu->iem.s.iNextMapping = 0;
335	Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
336	Assert(pVCpu->iem.s.fBypassHandlers == false);
337
338	#ifdef DBGFTRACE_ENABLED
339	switch (enmMode)
340	{
341	case IEMMODE_64BIT:
342	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
343	break;
344	case IEMMODE_32BIT:
345	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
346	break;
347	case IEMMODE_16BIT:
348	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
349	break;
350	}
351	#endif
352	}
353
354
355
356	/**
357	* Prefetch opcodes the first time when starting executing.
358	*
359	* @returns Strict VBox status code.
360	* @param pVCpu The cross context virtual CPU structure of the
361	* calling thread.
362	* @param fBypassHandlers Whether to bypass access handlers.
363	* @param fDisregardLock Whether to disregard LOCK prefixes.
364	*
365	* @todo Combine fDisregardLock and fBypassHandlers into a flag parameter and
366	* store them as such.
367	*/
368	static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) RT_NOEXCEPT
369	{
370	iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock);
371
372	#ifdef IEM_WITH_CODE_TLB
373	/** @todo Do ITLB lookup here. */
374
375	#else /* !IEM_WITH_CODE_TLB */
376
377	/*
378	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
379	*
380	* First translate CS:rIP to a physical address.
381	*/
382	uint32_t cbToTryRead;
383	RTGCPTR GCPtrPC;
384	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
385	{
386	cbToTryRead = GUEST_PAGE_SIZE;
387	GCPtrPC = pVCpu->cpum.GstCtx.rip;
388	if (IEM_IS_CANONICAL(GCPtrPC))
389	cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
390	else
391	return iemRaiseGeneralProtectionFault0(pVCpu);
392	}
393	else
394	{
395	uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
396	AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
397	if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
398	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
399	else
400	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
401	if (cbToTryRead) { /* likely */ }
402	else /* overflowed */
403	{
404	Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
405	cbToTryRead = UINT32_MAX;
406	}
407	GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
408	Assert(GCPtrPC <= UINT32_MAX);
409	}
410
411	PGMPTWALK Walk;
412	int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
413	if (RT_SUCCESS(rc))
414	Assert(Walk.fSucceeded); /* probable. */
415	else
416	{
417	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
418	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
419	if (Walk.fFailed & PGM_WALKFAIL_EPT)
420	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
421	#endif
422	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
423	}
424	if ((Walk.fEffective & X86_PTE_US) \|\| pVCpu->iem.s.uCpl != 3) { /* likely */ }
425	else
426	{
427	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
428	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
429	if (Walk.fFailed & PGM_WALKFAIL_EPT)
430	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
431	#endif
432	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
433	}
434	if (!(Walk.fEffective & X86_PTE_PAE_NX) \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
435	else
436	{
437	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
438	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
439	if (Walk.fFailed & PGM_WALKFAIL_EPT)
440	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
441	#endif
442	return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
443	}
444	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
445	/** @todo Check reserved bits and such stuff. PGM is better at doing
446	* that, so do it when implementing the guest virtual address
447	* TLB... */
448
449	/*
450	* Read the bytes at this address.
451	*/
452	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
453	if (cbToTryRead > cbLeftOnPage)
454	cbToTryRead = cbLeftOnPage;
455	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
456	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
457
458	if (!pVCpu->iem.s.fBypassHandlers)
459	{
460	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
461	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
462	{ /* likely */ }
463	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
464	{
465	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
466	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
467	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
468	}
469	else
470	{
471	Log((RT_SUCCESS(rcStrict)
472	? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
473	: "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
474	GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
475	return rcStrict;
476	}
477	}
478	else
479	{
480	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
481	if (RT_SUCCESS(rc))
482	{ /* likely */ }
483	else
484	{
485	Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
486	GCPtrPC, GCPhys, rc, cbToTryRead));
487	return rc;
488	}
489	}
490	pVCpu->iem.s.cbOpcode = cbToTryRead;
491	#endif /* !IEM_WITH_CODE_TLB */
492	return VINF_SUCCESS;
493	}
494
495
496	/**
497	* Invalidates the IEM TLBs.
498	*
499	* This is called internally as well as by PGM when moving GC mappings.
500	*
501	* @returns
502	* @param pVCpu The cross context virtual CPU structure of the calling
503	* thread.
504	*/
505	VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)
506	{
507	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
508	Log10(("IEMTlbInvalidateAll\n"));
509	# ifdef IEM_WITH_CODE_TLB
510	pVCpu->iem.s.cbInstrBufTotal = 0;
511	pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
512	if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
513	{ /* very likely */ }
514	else
515	{
516	pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
517	unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
518	while (i-- > 0)
519	pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
520	}
521	# endif
522
523	# ifdef IEM_WITH_DATA_TLB
524	pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
525	if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
526	{ /* very likely */ }
527	else
528	{
529	pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
530	unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
531	while (i-- > 0)
532	pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
533	}
534	# endif
535	#else
536	RT_NOREF(pVCpu);
537	#endif
538	}
539
540
541	/**
542	* Invalidates a page in the TLBs.
543	*
544	* @param pVCpu The cross context virtual CPU structure of the calling
545	* thread.
546	* @param GCPtr The address of the page to invalidate
547	* @thread EMT(pVCpu)
548	*/
549	VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
550	{
551	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
552	Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));
553	GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);
554	Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));
555	uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr);
556
557	# ifdef IEM_WITH_CODE_TLB
558	if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.CodeTlb.uTlbRevision))
559	{
560	pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
561	if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc))
562	pVCpu->iem.s.cbInstrBufTotal = 0;
563	}
564	# endif
565
566	# ifdef IEM_WITH_DATA_TLB
567	if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr \| pVCpu->iem.s.DataTlb.uTlbRevision))
568	pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
569	# endif
570	#else
571	NOREF(pVCpu); NOREF(GCPtr);
572	#endif
573	}
574
575
576	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
577	/**
578	* Invalid both TLBs slow fashion following a rollover.
579	*
580	* Worker for IEMTlbInvalidateAllPhysical,
581	* IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,
582	* iemMemMapJmp and others.
583	*
584	* @thread EMT(pVCpu)
585	*/
586	static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu)
587	{
588	Log10(("IEMTlbInvalidateAllPhysicalSlow\n"));
589	ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
590	ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
591
592	unsigned i;
593	# ifdef IEM_WITH_CODE_TLB
594	i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
595	while (i-- > 0)
596	{
597	pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
598	pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
599	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
600	}
601	# endif
602	# ifdef IEM_WITH_DATA_TLB
603	i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
604	while (i-- > 0)
605	{
606	pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
607	pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~( IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ
608	\| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PHYS_REV);
609	}
610	# endif
611
612	}
613	#endif
614
615
616	/**
617	* Invalidates the host physical aspects of the IEM TLBs.
618	*
619	* This is called internally as well as by PGM when moving GC mappings.
620	*
621	* @param pVCpu The cross context virtual CPU structure of the calling
622	* thread.
623	* @note Currently not used.
624	*/
625	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
626	{
627	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
628	/* Note! This probably won't end up looking exactly like this, but it give an idea... */
629	Log10(("IEMTlbInvalidateAllPhysical\n"));
630
631	# ifdef IEM_WITH_CODE_TLB
632	pVCpu->iem.s.cbInstrBufTotal = 0;
633	# endif
634	uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
635	if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))
636	{
637	pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
638	pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
639	}
640	else
641	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
642	#else
643	NOREF(pVCpu);
644	#endif
645	}
646
647
648	/**
649	* Invalidates the host physical aspects of the IEM TLBs.
650	*
651	* This is called internally as well as by PGM when moving GC mappings.
652	*
653	* @param pVM The cross context VM structure.
654	* @param idCpuCaller The ID of the calling EMT if available to the caller,
655	* otherwise NIL_VMCPUID.
656	*
657	* @remarks Caller holds the PGM lock.
658	*/
659	VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller)
660	{
661	#if defined(IEM_WITH_CODE_TLB) \|\| defined(IEM_WITH_DATA_TLB)
662	PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);
663	if (pVCpuCaller)
664	VMCPU_ASSERT_EMT(pVCpuCaller);
665	Log10(("IEMTlbInvalidateAllPhysicalAllCpus\n"));
666
667	VMCC_FOR_EACH_VMCPU(pVM)
668	{
669	# ifdef IEM_WITH_CODE_TLB
670	if (pVCpuCaller == pVCpu)
671	pVCpu->iem.s.cbInstrBufTotal = 0;
672	# endif
673
674	uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);
675	uint64_t uTlbPhysRevNew = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;
676	if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))
677	{ /* likely */}
678	else if (pVCpuCaller == pVCpu)
679	uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;
680	else
681	{
682	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
683	continue;
684	}
685	ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
686	ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
687	}
688	VMCC_FOR_EACH_VMCPU_END(pVM);
689
690	#else
691	RT_NOREF(pVM, idCpuCaller);
692	#endif
693	}
694
695	#ifdef IEM_WITH_CODE_TLB
696
697	/**
698	* Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
699	* failure and jumps.
700	*
701	* We end up here for a number of reasons:
702	* - pbInstrBuf isn't yet initialized.
703	* - Advancing beyond the buffer boundrary (e.g. cross page).
704	* - Advancing beyond the CS segment limit.
705	* - Fetching from non-mappable page (e.g. MMIO).
706	*
707	* @param pVCpu The cross context virtual CPU structure of the
708	* calling thread.
709	* @param pvDst Where to return the bytes.
710	* @param cbDst Number of bytes to read.
711	*
712	* @todo Make cbDst = 0 a way of initializing pbInstrBuf?
713	*/
714	void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) RT_NOEXCEPT
715	{
716	#ifdef IN_RING3
717	for (;;)
718	{
719	Assert(cbDst <= 8);
720	uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
721
722	/*
723	* We might have a partial buffer match, deal with that first to make the
724	* rest simpler. This is the first part of the cross page/buffer case.
725	*/
726	if (pVCpu->iem.s.pbInstrBuf != NULL)
727	{
728	if (offBuf < pVCpu->iem.s.cbInstrBuf)
729	{
730	Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
731	uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
732	memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
733
734	cbDst -= cbCopy;
735	pvDst = (uint8_t *)pvDst + cbCopy;
736	offBuf += cbCopy;
737	pVCpu->iem.s.offInstrNextByte += offBuf;
738	}
739	}
740
741	/*
742	* Check segment limit, figuring how much we're allowed to access at this point.
743	*
744	* We will fault immediately if RIP is past the segment limit / in non-canonical
745	* territory. If we do continue, there are one or more bytes to read before we
746	* end up in trouble and we need to do that first before faulting.
747	*/
748	RTGCPTR GCPtrFirst;
749	uint32_t cbMaxRead;
750	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
751	{
752	GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
753	if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
754	{ /* likely */ }
755	else
756	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
757	cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
758	}
759	else
760	{
761	GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
762	Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
763	if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
764	{ /* likely */ }
765	else
766	iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
767	cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
768	if (cbMaxRead != 0)
769	{ /* likely */ }
770	else
771	{
772	/* Overflowed because address is 0 and limit is max. */
773	Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
774	cbMaxRead = X86_PAGE_SIZE;
775	}
776	GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
777	uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
778	if (cbMaxRead2 < cbMaxRead)
779	cbMaxRead = cbMaxRead2;
780	/** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
781	}
782
783	/*
784	* Get the TLB entry for this piece of code.
785	*/
786	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.CodeTlb, GCPtrFirst);
787	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag);
788	if (pTlbe->uTag == uTag)
789	{
790	/* likely when executing lots of code, otherwise unlikely */
791	# ifdef VBOX_WITH_STATISTICS
792	pVCpu->iem.s.CodeTlb.cTlbHits++;
793	# endif
794	}
795	else
796	{
797	pVCpu->iem.s.CodeTlb.cTlbMisses++;
798	PGMPTWALK Walk;
799	int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
800	if (RT_FAILURE(rc))
801	{
802	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
803	/** @todo Nested VMX: Need to handle EPT violation/misconfig here? */
804	Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
805	#endif
806	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
807	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
808	}
809
810	AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
811	Assert(Walk.fSucceeded);
812	pTlbe->uTag = uTag;
813	pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A))
814	\| (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
815	pTlbe->GCPhys = Walk.GCPhys;
816	pTlbe->pbMappingR3 = NULL;
817	}
818
819	/*
820	* Check TLB page table level access flags.
821	*/
822	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_EXEC))
823	{
824	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
825	{
826	Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
827	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
828	}
829	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
830	{
831	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
832	iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
833	}
834	}
835
836	/*
837	* Look up the physical page info if necessary.
838	*/
839	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
840	{ /* not necessary */ }
841	else
842	{
843	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
844	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
845	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
846	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
847	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
848	{ /* likely */ }
849	else
850	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
851	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
852	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
853	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
854	&pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
855	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
856	}
857
858	# if defined(IN_RING3) \|\| defined(IN_RING0) /** @todo fixme */
859	/*
860	* Try do a direct read using the pbMappingR3 pointer.
861	*/
862	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ))
863	== pVCpu->iem.s.CodeTlb.uTlbPhysRev)
864	{
865	uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
866	pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
867	if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
868	{
869	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
870	pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
871	}
872	else
873	{
874	uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
875	Assert(cbInstr < cbMaxRead);
876	pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
877	pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
878	}
879	if (cbDst <= cbMaxRead)
880	{
881	pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
882	pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
883	pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
884	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
885	return;
886	}
887	pVCpu->iem.s.pbInstrBuf = NULL;
888
889	memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
890	pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
891	}
892	else
893	# endif
894	#if 0
895	/*
896	* If there is no special read handling, so we can read a bit more and
897	* put it in the prefetch buffer.
898	*/
899	if ( cbDst < cbMaxRead
900	&& (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
901	{
902	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
903	&pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
904	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
905	{ /* likely */ }
906	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
907	{
908	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
909	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
910	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
911	AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
912	}
913	else
914	{
915	Log((RT_SUCCESS(rcStrict)
916	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
917	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
918	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
919	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
920	}
921	}
922	/*
923	* Special read handling, so only read exactly what's needed.
924	* This is a highly unlikely scenario.
925	*/
926	else
927	#endif
928	{
929	pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
930	uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
931	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
932	pvDst, cbToRead, PGMACCESSORIGIN_IEM);
933	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
934	{ /* likely */ }
935	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
936	{
937	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
938	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
939	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
940	AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
941	}
942	else
943	{
944	Log((RT_SUCCESS(rcStrict)
945	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
946	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
947	GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
948	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
949	}
950	pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
951	if (cbToRead == cbDst)
952	return;
953	}
954
955	/*
956	* More to read, loop.
957	*/
958	cbDst -= cbMaxRead;
959	pvDst = (uint8_t *)pvDst + cbMaxRead;
960	}
961	#else
962	RT_NOREF(pvDst, cbDst);
963	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
964	#endif
965	}
966
967	#else
968
969	/**
970	* Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
971	* exception if it fails.
972	*
973	* @returns Strict VBox status code.
974	* @param pVCpu The cross context virtual CPU structure of the
975	* calling thread.
976	* @param cbMin The minimum number of bytes relative offOpcode
977	* that must be read.
978	*/
979	VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
980	{
981	/*
982	* What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
983	*
984	* First translate CS:rIP to a physical address.
985	*/
986	uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
987	uint32_t cbToTryRead;
988	RTGCPTR GCPtrNext;
989	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
990	{
991	cbToTryRead = GUEST_PAGE_SIZE;
992	GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
993	if (!IEM_IS_CANONICAL(GCPtrNext))
994	return iemRaiseGeneralProtectionFault0(pVCpu);
995	}
996	else
997	{
998	uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
999	Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) \|\| pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1000	GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1001	if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1002	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1003	cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1004	if (!cbToTryRead) /* overflowed */
1005	{
1006	Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1007	cbToTryRead = UINT32_MAX;
1008	/** @todo check out wrapping around the code segment. */
1009	}
1010	if (cbToTryRead < cbMin - cbLeft)
1011	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1012	GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1013	}
1014
1015	/* Only read up to the end of the page, and make sure we don't read more
1016	than the opcode buffer can hold. */
1017	uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1018	if (cbToTryRead > cbLeftOnPage)
1019	cbToTryRead = cbLeftOnPage;
1020	if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
1021	cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
1022	/** @todo r=bird: Convert assertion into undefined opcode exception? */
1023	Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1024
1025	PGMPTWALK Walk;
1026	int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
1027	if (RT_FAILURE(rc))
1028	{
1029	Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1030	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1031	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1032	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1033	#endif
1034	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1035	}
1036	if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
1037	{
1038	Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
1039	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1040	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1041	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1042	#endif
1043	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1044	}
1045	if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1046	{
1047	Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
1048	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1049	if (Walk.fFailed & PGM_WALKFAIL_EPT)
1050	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1051	#endif
1052	return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1053	}
1054	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1055	Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
1056	/** @todo Check reserved bits and such stuff. PGM is better at doing
1057	* that, so do it when implementing the guest virtual address
1058	* TLB... */
1059
1060	/*
1061	* Read the bytes at this address.
1062	*
1063	* We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
1064	* and since PATM should only patch the start of an instruction there
1065	* should be no need to check again here.
1066	*/
1067	if (!pVCpu->iem.s.fBypassHandlers)
1068	{
1069	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
1070	cbToTryRead, PGMACCESSORIGIN_IEM);
1071	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1072	{ /* likely */ }
1073	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1074	{
1075	Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1076	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1077	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1078	}
1079	else
1080	{
1081	Log((RT_SUCCESS(rcStrict)
1082	? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1083	: "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1084	GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1085	return rcStrict;
1086	}
1087	}
1088	else
1089	{
1090	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
1091	if (RT_SUCCESS(rc))
1092	{ /* likely */ }
1093	else
1094	{
1095	Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
1096	return rc;
1097	}
1098	}
1099	pVCpu->iem.s.cbOpcode += cbToTryRead;
1100	Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
1101
1102	return VINF_SUCCESS;
1103	}
1104
1105	#endif /* !IEM_WITH_CODE_TLB */
1106	#ifndef IEM_WITH_SETJMP
1107
1108	/**
1109	* Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
1110	*
1111	* @returns Strict VBox status code.
1112	* @param pVCpu The cross context virtual CPU structure of the
1113	* calling thread.
1114	* @param pb Where to return the opcode byte.
1115	*/
1116	VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
1117	{
1118	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1119	if (rcStrict == VINF_SUCCESS)
1120	{
1121	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1122	*pb = pVCpu->iem.s.abOpcode[offOpcode];
1123	pVCpu->iem.s.offOpcode = offOpcode + 1;
1124	}
1125	else
1126	*pb = 0;
1127	return rcStrict;
1128	}
1129
1130	#else /* IEM_WITH_SETJMP */
1131
1132	/**
1133	* Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
1134	*
1135	* @returns The opcode byte.
1136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1137	*/
1138	uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1139	{
1140	# ifdef IEM_WITH_CODE_TLB
1141	uint8_t u8;
1142	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
1143	return u8;
1144	# else
1145	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
1146	if (rcStrict == VINF_SUCCESS)
1147	return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
1148	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1149	# endif
1150	}
1151
1152	#endif /* IEM_WITH_SETJMP */
1153
1154	#ifndef IEM_WITH_SETJMP
1155
1156	/**
1157	* Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
1158	*
1159	* @returns Strict VBox status code.
1160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1161	* @param pu16 Where to return the opcode dword.
1162	*/
1163	VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1164	{
1165	uint8_t u8;
1166	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1167	if (rcStrict == VINF_SUCCESS)
1168	*pu16 = (int8_t)u8;
1169	return rcStrict;
1170	}
1171
1172
1173	/**
1174	* Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
1175	*
1176	* @returns Strict VBox status code.
1177	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1178	* @param pu32 Where to return the opcode dword.
1179	*/
1180	VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1181	{
1182	uint8_t u8;
1183	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1184	if (rcStrict == VINF_SUCCESS)
1185	*pu32 = (int8_t)u8;
1186	return rcStrict;
1187	}
1188
1189
1190	/**
1191	* Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
1192	*
1193	* @returns Strict VBox status code.
1194	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1195	* @param pu64 Where to return the opcode qword.
1196	*/
1197	VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1198	{
1199	uint8_t u8;
1200	VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
1201	if (rcStrict == VINF_SUCCESS)
1202	*pu64 = (int8_t)u8;
1203	return rcStrict;
1204	}
1205
1206	#endif /* !IEM_WITH_SETJMP */
1207
1208
1209	#ifndef IEM_WITH_SETJMP
1210
1211	/**
1212	* Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
1213	*
1214	* @returns Strict VBox status code.
1215	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1216	* @param pu16 Where to return the opcode word.
1217	*/
1218	VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
1219	{
1220	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1221	if (rcStrict == VINF_SUCCESS)
1222	{
1223	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1224	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1225	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1226	# else
1227	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1228	# endif
1229	pVCpu->iem.s.offOpcode = offOpcode + 2;
1230	}
1231	else
1232	*pu16 = 0;
1233	return rcStrict;
1234	}
1235
1236	#else /* IEM_WITH_SETJMP */
1237
1238	/**
1239	* Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
1240	*
1241	* @returns The opcode word.
1242	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1243	*/
1244	uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1245	{
1246	# ifdef IEM_WITH_CODE_TLB
1247	uint16_t u16;
1248	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
1249	return u16;
1250	# else
1251	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1252	if (rcStrict == VINF_SUCCESS)
1253	{
1254	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1255	pVCpu->iem.s.offOpcode += 2;
1256	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1257	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1258	# else
1259	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1260	# endif
1261	}
1262	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1263	# endif
1264	}
1265
1266	#endif /* IEM_WITH_SETJMP */
1267
1268	#ifndef IEM_WITH_SETJMP
1269
1270	/**
1271	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
1272	*
1273	* @returns Strict VBox status code.
1274	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1275	* @param pu32 Where to return the opcode double word.
1276	*/
1277	VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1278	{
1279	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1280	if (rcStrict == VINF_SUCCESS)
1281	{
1282	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1283	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1284	pVCpu->iem.s.offOpcode = offOpcode + 2;
1285	}
1286	else
1287	*pu32 = 0;
1288	return rcStrict;
1289	}
1290
1291
1292	/**
1293	* Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
1294	*
1295	* @returns Strict VBox status code.
1296	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1297	* @param pu64 Where to return the opcode quad word.
1298	*/
1299	VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1300	{
1301	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
1302	if (rcStrict == VINF_SUCCESS)
1303	{
1304	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1305	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1306	pVCpu->iem.s.offOpcode = offOpcode + 2;
1307	}
1308	else
1309	*pu64 = 0;
1310	return rcStrict;
1311	}
1312
1313	#endif /* !IEM_WITH_SETJMP */
1314
1315	#ifndef IEM_WITH_SETJMP
1316
1317	/**
1318	* Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
1319	*
1320	* @returns Strict VBox status code.
1321	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1322	* @param pu32 Where to return the opcode dword.
1323	*/
1324	VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1325	{
1326	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1327	if (rcStrict == VINF_SUCCESS)
1328	{
1329	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1330	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1331	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1332	# else
1333	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1334	pVCpu->iem.s.abOpcode[offOpcode + 1],
1335	pVCpu->iem.s.abOpcode[offOpcode + 2],
1336	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1337	# endif
1338	pVCpu->iem.s.offOpcode = offOpcode + 4;
1339	}
1340	else
1341	*pu32 = 0;
1342	return rcStrict;
1343	}
1344
1345	#else /* IEM_WITH_SETJMP */
1346
1347	/**
1348	* Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
1349	*
1350	* @returns The opcode dword.
1351	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1352	*/
1353	uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1354	{
1355	# ifdef IEM_WITH_CODE_TLB
1356	uint32_t u32;
1357	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
1358	return u32;
1359	# else
1360	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1361	if (rcStrict == VINF_SUCCESS)
1362	{
1363	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1364	pVCpu->iem.s.offOpcode = offOpcode + 4;
1365	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1366	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1367	# else
1368	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1369	pVCpu->iem.s.abOpcode[offOpcode + 1],
1370	pVCpu->iem.s.abOpcode[offOpcode + 2],
1371	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1372	# endif
1373	}
1374	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1375	# endif
1376	}
1377
1378	#endif /* IEM_WITH_SETJMP */
1379
1380	#ifndef IEM_WITH_SETJMP
1381
1382	/**
1383	* Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
1384	*
1385	* @returns Strict VBox status code.
1386	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1387	* @param pu64 Where to return the opcode dword.
1388	*/
1389	VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1390	{
1391	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1392	if (rcStrict == VINF_SUCCESS)
1393	{
1394	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1395	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1396	pVCpu->iem.s.abOpcode[offOpcode + 1],
1397	pVCpu->iem.s.abOpcode[offOpcode + 2],
1398	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1399	pVCpu->iem.s.offOpcode = offOpcode + 4;
1400	}
1401	else
1402	*pu64 = 0;
1403	return rcStrict;
1404	}
1405
1406
1407	/**
1408	* Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
1409	*
1410	* @returns Strict VBox status code.
1411	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1412	* @param pu64 Where to return the opcode qword.
1413	*/
1414	VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1415	{
1416	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
1417	if (rcStrict == VINF_SUCCESS)
1418	{
1419	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1420	*pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1421	pVCpu->iem.s.abOpcode[offOpcode + 1],
1422	pVCpu->iem.s.abOpcode[offOpcode + 2],
1423	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1424	pVCpu->iem.s.offOpcode = offOpcode + 4;
1425	}
1426	else
1427	*pu64 = 0;
1428	return rcStrict;
1429	}
1430
1431	#endif /* !IEM_WITH_SETJMP */
1432
1433	#ifndef IEM_WITH_SETJMP
1434
1435	/**
1436	* Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
1437	*
1438	* @returns Strict VBox status code.
1439	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1440	* @param pu64 Where to return the opcode qword.
1441	*/
1442	VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1443	{
1444	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1445	if (rcStrict == VINF_SUCCESS)
1446	{
1447	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1448	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1449	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1450	# else
1451	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1452	pVCpu->iem.s.abOpcode[offOpcode + 1],
1453	pVCpu->iem.s.abOpcode[offOpcode + 2],
1454	pVCpu->iem.s.abOpcode[offOpcode + 3],
1455	pVCpu->iem.s.abOpcode[offOpcode + 4],
1456	pVCpu->iem.s.abOpcode[offOpcode + 5],
1457	pVCpu->iem.s.abOpcode[offOpcode + 6],
1458	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1459	# endif
1460	pVCpu->iem.s.offOpcode = offOpcode + 8;
1461	}
1462	else
1463	*pu64 = 0;
1464	return rcStrict;
1465	}
1466
1467	#else /* IEM_WITH_SETJMP */
1468
1469	/**
1470	* Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
1471	*
1472	* @returns The opcode qword.
1473	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1474	*/
1475	uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
1476	{
1477	# ifdef IEM_WITH_CODE_TLB
1478	uint64_t u64;
1479	iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
1480	return u64;
1481	# else
1482	VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
1483	if (rcStrict == VINF_SUCCESS)
1484	{
1485	uint8_t offOpcode = pVCpu->iem.s.offOpcode;
1486	pVCpu->iem.s.offOpcode = offOpcode + 8;
1487	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1488	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1489	# else
1490	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1491	pVCpu->iem.s.abOpcode[offOpcode + 1],
1492	pVCpu->iem.s.abOpcode[offOpcode + 2],
1493	pVCpu->iem.s.abOpcode[offOpcode + 3],
1494	pVCpu->iem.s.abOpcode[offOpcode + 4],
1495	pVCpu->iem.s.abOpcode[offOpcode + 5],
1496	pVCpu->iem.s.abOpcode[offOpcode + 6],
1497	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1498	# endif
1499	}
1500	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1501	# endif
1502	}
1503
1504	#endif /* IEM_WITH_SETJMP */
1505
1506
1507
1508	/** @name Misc Worker Functions.
1509	* @{
1510	*/
1511
1512	/**
1513	* Gets the exception class for the specified exception vector.
1514	*
1515	* @returns The class of the specified exception.
1516	* @param uVector The exception vector.
1517	*/
1518	static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT
1519	{
1520	Assert(uVector <= X86_XCPT_LAST);
1521	switch (uVector)
1522	{
1523	case X86_XCPT_DE:
1524	case X86_XCPT_TS:
1525	case X86_XCPT_NP:
1526	case X86_XCPT_SS:
1527	case X86_XCPT_GP:
1528	case X86_XCPT_SX: /* AMD only */
1529	return IEMXCPTCLASS_CONTRIBUTORY;
1530
1531	case X86_XCPT_PF:
1532	case X86_XCPT_VE: /* Intel only */
1533	return IEMXCPTCLASS_PAGE_FAULT;
1534
1535	case X86_XCPT_DF:
1536	return IEMXCPTCLASS_DOUBLE_FAULT;
1537	}
1538	return IEMXCPTCLASS_BENIGN;
1539	}
1540
1541
1542	/**
1543	* Evaluates how to handle an exception caused during delivery of another event
1544	* (exception / interrupt).
1545	*
1546	* @returns How to handle the recursive exception.
1547	* @param pVCpu The cross context virtual CPU structure of the
1548	* calling thread.
1549	* @param fPrevFlags The flags of the previous event.
1550	* @param uPrevVector The vector of the previous event.
1551	* @param fCurFlags The flags of the current exception.
1552	* @param uCurVector The vector of the current exception.
1553	* @param pfXcptRaiseInfo Where to store additional information about the
1554	* exception condition. Optional.
1555	*/
1556	VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
1557	uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
1558	{
1559	/*
1560	* Only CPU exceptions can be raised while delivering other events, software interrupt
1561	* (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
1562	*/
1563	AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
1564	Assert(pVCpu); RT_NOREF(pVCpu);
1565	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
1566
1567	IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
1568	IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
1569	if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1570	{
1571	IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
1572	if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
1573	{
1574	IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
1575	if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
1576	&& ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
1577	\|\| enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
1578	{
1579	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1580	fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
1581	: IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
1582	Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
1583	uCurVector, pVCpu->cpum.GstCtx.cr2));
1584	}
1585	else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1586	&& enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
1587	{
1588	enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
1589	Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
1590	}
1591	else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
1592	&& ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
1593	\|\| enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
1594	{
1595	enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
1596	Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
1597	}
1598	}
1599	else
1600	{
1601	if (uPrevVector == X86_XCPT_NMI)
1602	{
1603	fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
1604	if (uCurVector == X86_XCPT_PF)
1605	{
1606	fRaiseInfo \|= IEMXCPTRAISEINFO_NMI_PF;
1607	Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
1608	}
1609	}
1610	else if ( uPrevVector == X86_XCPT_AC
1611	&& uCurVector == X86_XCPT_AC)
1612	{
1613	enmRaise = IEMXCPTRAISE_CPU_HANG;
1614	fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
1615	Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
1616	}
1617	}
1618	}
1619	else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
1620	{
1621	fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
1622	if (uCurVector == X86_XCPT_PF)
1623	fRaiseInfo \|= IEMXCPTRAISEINFO_EXT_INT_PF;
1624	}
1625	else
1626	{
1627	Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
1628	fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
1629	}
1630
1631	if (pfXcptRaiseInfo)
1632	*pfXcptRaiseInfo = fRaiseInfo;
1633	return enmRaise;
1634	}
1635
1636
1637	/**
1638	* Enters the CPU shutdown state initiated by a triple fault or other
1639	* unrecoverable conditions.
1640	*
1641	* @returns Strict VBox status code.
1642	* @param pVCpu The cross context virtual CPU structure of the
1643	* calling thread.
1644	*/
1645	static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT
1646	{
1647	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
1648	IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
1649
1650	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
1651	{
1652	Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
1653	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
1654	}
1655
1656	RT_NOREF(pVCpu);
1657	return VINF_EM_TRIPLE_FAULT;
1658	}
1659
1660
1661	/**
1662	* Validates a new SS segment.
1663	*
1664	* @returns VBox strict status code.
1665	* @param pVCpu The cross context virtual CPU structure of the
1666	* calling thread.
1667	* @param NewSS The new SS selctor.
1668	* @param uCpl The CPL to load the stack for.
1669	* @param pDesc Where to return the descriptor.
1670	*/
1671	static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT
1672	{
1673	/* Null selectors are not allowed (we're not called for dispatching
1674	interrupts with SS=0 in long mode). */
1675	if (!(NewSS & X86_SEL_MASK_OFF_RPL))
1676	{
1677	Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
1678	return iemRaiseTaskSwitchFault0(pVCpu);
1679	}
1680
1681	/** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
1682	if ((NewSS & X86_SEL_RPL) != uCpl)
1683	{
1684	Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
1685	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1686	}
1687
1688	/*
1689	* Read the descriptor.
1690	*/
1691	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
1692	if (rcStrict != VINF_SUCCESS)
1693	return rcStrict;
1694
1695	/*
1696	* Perform the descriptor validation documented for LSS, POP SS and MOV SS.
1697	*/
1698	if (!pDesc->Legacy.Gen.u1DescType)
1699	{
1700	Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1701	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1702	}
1703
1704	if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
1705	\|\| !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
1706	{
1707	Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
1708	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1709	}
1710	if (pDesc->Legacy.Gen.u2Dpl != uCpl)
1711	{
1712	Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
1713	return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
1714	}
1715
1716	/* Is it there? */
1717	/** @todo testcase: Is this checked before the canonical / limit check below? */
1718	if (!pDesc->Legacy.Gen.u1Present)
1719	{
1720	Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
1721	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
1722	}
1723
1724	return VINF_SUCCESS;
1725	}
1726
1727	/** @} */
1728
1729
1730	/** @name Raising Exceptions.
1731	*
1732	* @{
1733	*/
1734
1735
1736	/**
1737	* Loads the specified stack far pointer from the TSS.
1738	*
1739	* @returns VBox strict status code.
1740	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1741	* @param uCpl The CPL to load the stack for.
1742	* @param pSelSS Where to return the new stack segment.
1743	* @param puEsp Where to return the new stack pointer.
1744	*/
1745	static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT
1746	{
1747	VBOXSTRICTRC rcStrict;
1748	Assert(uCpl < 4);
1749
1750	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1751	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
1752	{
1753	/*
1754	* 16-bit TSS (X86TSS16).
1755	*/
1756	case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1757	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
1758	{
1759	uint32_t off = uCpl * 4 + 2;
1760	if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1761	{
1762	/** @todo check actual access pattern here. */
1763	uint32_t u32Tmp = 0; /* gcc maybe... */
1764	rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1765	if (rcStrict == VINF_SUCCESS)
1766	{
1767	*puEsp = RT_LOWORD(u32Tmp);
1768	*pSelSS = RT_HIWORD(u32Tmp);
1769	return VINF_SUCCESS;
1770	}
1771	}
1772	else
1773	{
1774	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1775	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1776	}
1777	break;
1778	}
1779
1780	/*
1781	* 32-bit TSS (X86TSS32).
1782	*/
1783	case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
1784	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
1785	{
1786	uint32_t off = uCpl * 8 + 4;
1787	if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
1788	{
1789	/** @todo check actual access pattern here. */
1790	uint64_t u64Tmp;
1791	rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1792	if (rcStrict == VINF_SUCCESS)
1793	{
1794	*puEsp = u64Tmp & UINT32_MAX;
1795	*pSelSS = (RTSEL)(u64Tmp >> 32);
1796	return VINF_SUCCESS;
1797	}
1798	}
1799	else
1800	{
1801	Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
1802	rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1803	}
1804	break;
1805	}
1806
1807	default:
1808	AssertFailed();
1809	rcStrict = VERR_IEM_IPE_4;
1810	break;
1811	}
1812
1813	puEsp = 0; / make gcc happy */
1814	pSelSS = 0; / make gcc happy */
1815	return rcStrict;
1816	}
1817
1818
1819	/**
1820	* Loads the specified stack pointer from the 64-bit TSS.
1821	*
1822	* @returns VBox strict status code.
1823	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1824	* @param uCpl The CPL to load the stack for.
1825	* @param uIst The interrupt stack table index, 0 if to use uCpl.
1826	* @param puRsp Where to return the new stack pointer.
1827	*/
1828	static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT
1829	{
1830	Assert(uCpl < 4);
1831	Assert(uIst < 8);
1832	puRsp = 0; / make gcc happy */
1833
1834	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
1835	AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
1836
1837	uint32_t off;
1838	if (uIst)
1839	off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
1840	else
1841	off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
1842	if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
1843	{
1844	Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
1845	return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
1846	}
1847
1848	return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
1849	}
1850
1851
1852	/**
1853	* Adjust the CPU state according to the exception being raised.
1854	*
1855	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1856	* @param u8Vector The exception that has been raised.
1857	*/
1858	DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
1859	{
1860	switch (u8Vector)
1861	{
1862	case X86_XCPT_DB:
1863	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
1864	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
1865	break;
1866	/** @todo Read the AMD and Intel exception reference... */
1867	}
1868	}
1869
1870
1871	/**
1872	* Implements exceptions and interrupts for real mode.
1873	*
1874	* @returns VBox strict status code.
1875	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1876	* @param cbInstr The number of bytes to offset rIP by in the return
1877	* address.
1878	* @param u8Vector The interrupt / exception vector number.
1879	* @param fFlags The flags.
1880	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
1881	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
1882	*/
1883	static VBOXSTRICTRC
1884	iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
1885	uint8_t cbInstr,
1886	uint8_t u8Vector,
1887	uint32_t fFlags,
1888	uint16_t uErr,
1889	uint64_t uCr2) RT_NOEXCEPT
1890	{
1891	NOREF(uErr); NOREF(uCr2);
1892	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
1893
1894	/*
1895	* Read the IDT entry.
1896	*/
1897	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
1898	{
1899	Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
1900	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
1901	}
1902	RTFAR16 Idte;
1903	VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t )&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) u8Vector);
1904	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1905	{
1906	Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
1907	return rcStrict;
1908	}
1909
1910	/*
1911	* Push the stack frame.
1912	*/
1913	uint16_t *pu16Frame;
1914	uint64_t uNewRsp;
1915	rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &uNewRsp);
1916	if (rcStrict != VINF_SUCCESS)
1917	return rcStrict;
1918
1919	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
1920	#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
1921	AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
1922	if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
1923	fEfl \|= UINT16_C(0xf000);
1924	#endif
1925	pu16Frame[2] = (uint16_t)fEfl;
1926	pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
1927	pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
1928	rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
1929	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
1930	return rcStrict;
1931
1932	/*
1933	* Load the vector address into cs:ip and make exception specific state
1934	* adjustments.
1935	*/
1936	pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
1937	pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
1938	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
1939	pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
1940	/** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
1941	pVCpu->cpum.GstCtx.rip = Idte.off;
1942	fEfl &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_AC);
1943	IEMMISC_SET_EFL(pVCpu, fEfl);
1944
1945	/** @todo do we actually do this in real mode? */
1946	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
1947	iemRaiseXcptAdjustState(pVCpu, u8Vector);
1948
1949	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
1950	}
1951
1952
1953	/**
1954	* Loads a NULL data selector into when coming from V8086 mode.
1955	*
1956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1957	* @param pSReg Pointer to the segment register.
1958	*/
1959	DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1960	{
1961	pSReg->Sel = 0;
1962	pSReg->ValidSel = 0;
1963	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1964	{
1965	/* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
1966	pSReg->Attr.u &= X86DESCATTR_DT \| X86DESCATTR_TYPE \| X86DESCATTR_DPL \| X86DESCATTR_G \| X86DESCATTR_D;
1967	pSReg->Attr.u \|= X86DESCATTR_UNUSABLE;
1968	}
1969	else
1970	{
1971	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1972	/** @todo check this on AMD-V */
1973	pSReg->u64Base = 0;
1974	pSReg->u32Limit = 0;
1975	}
1976	}
1977
1978
1979	/**
1980	* Loads a segment selector during a task switch in V8086 mode.
1981	*
1982	* @param pSReg Pointer to the segment register.
1983	* @param uSel The selector value to load.
1984	*/
1985	DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
1986	{
1987	/* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
1988	pSReg->Sel = uSel;
1989	pSReg->ValidSel = uSel;
1990	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1991	pSReg->u64Base = uSel << 4;
1992	pSReg->u32Limit = 0xffff;
1993	pSReg->Attr.u = 0xf3;
1994	}
1995
1996
1997	/**
1998	* Loads a segment selector during a task switch in protected mode.
1999	*
2000	* In this task switch scenario, we would throw \#TS exceptions rather than
2001	* \#GPs.
2002	*
2003	* @returns VBox strict status code.
2004	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2005	* @param pSReg Pointer to the segment register.
2006	* @param uSel The new selector value.
2007	*
2008	* @remarks This does _not_ handle CS or SS.
2009	* @remarks This expects pVCpu->iem.s.uCpl to be up to date.
2010	*/
2011	static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT
2012	{
2013	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2014
2015	/* Null data selector. */
2016	if (!(uSel & X86_SEL_MASK_OFF_RPL))
2017	{
2018	iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
2019	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2020	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2021	return VINF_SUCCESS;
2022	}
2023
2024	/* Fetch the descriptor. */
2025	IEMSELDESC Desc;
2026	VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
2027	if (rcStrict != VINF_SUCCESS)
2028	{
2029	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
2030	VBOXSTRICTRC_VAL(rcStrict)));
2031	return rcStrict;
2032	}
2033
2034	/* Must be a data segment or readable code segment. */
2035	if ( !Desc.Legacy.Gen.u1DescType
2036	\|\| (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2037	{
2038	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
2039	Desc.Legacy.Gen.u4Type));
2040	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2041	}
2042
2043	/* Check privileges for data segments and non-conforming code segments. */
2044	if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2045	!= (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_CONF))
2046	{
2047	/* The RPL and the new CPL must be less than or equal to the DPL. */
2048	if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
2049	\|\| (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
2050	{
2051	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
2052	uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
2053	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2054	}
2055	}
2056
2057	/* Is it there? */
2058	if (!Desc.Legacy.Gen.u1Present)
2059	{
2060	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
2061	return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
2062	}
2063
2064	/* The base and limit. */
2065	uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
2066	uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
2067
2068	/*
2069	* Ok, everything checked out fine. Now set the accessed bit before
2070	* committing the result into the registers.
2071	*/
2072	if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2073	{
2074	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
2075	if (rcStrict != VINF_SUCCESS)
2076	return rcStrict;
2077	Desc.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2078	}
2079
2080	/* Commit */
2081	pSReg->Sel = uSel;
2082	pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
2083	pSReg->u32Limit = cbLimit;
2084	pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
2085	pSReg->ValidSel = uSel;
2086	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
2087	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2088	pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
2089
2090	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
2091	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2092	return VINF_SUCCESS;
2093	}
2094
2095
2096	/**
2097	* Performs a task switch.
2098	*
2099	* If the task switch is the result of a JMP, CALL or IRET instruction, the
2100	* caller is responsible for performing the necessary checks (like DPL, TSS
2101	* present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
2102	* reference for JMP, CALL, IRET.
2103	*
2104	* If the task switch is the due to a software interrupt or hardware exception,
2105	* the caller is responsible for validating the TSS selector and descriptor. See
2106	* Intel Instruction reference for INT n.
2107	*
2108	* @returns VBox strict status code.
2109	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2110	* @param enmTaskSwitch The cause of the task switch.
2111	* @param uNextEip The EIP effective after the task switch.
2112	* @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
2113	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2114	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2115	* @param SelTSS The TSS selector of the new task.
2116	* @param pNewDescTSS Pointer to the new TSS descriptor.
2117	*/
2118	VBOXSTRICTRC
2119	iemTaskSwitch(PVMCPUCC pVCpu,
2120	IEMTASKSWITCH enmTaskSwitch,
2121	uint32_t uNextEip,
2122	uint32_t fFlags,
2123	uint16_t uErr,
2124	uint64_t uCr2,
2125	RTSEL SelTSS,
2126	PIEMSELDESC pNewDescTSS) RT_NOEXCEPT
2127	{
2128	Assert(!IEM_IS_REAL_MODE(pVCpu));
2129	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
2130	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2131
2132	uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
2133	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2134	\|\| uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2135	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2136	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2137
2138	bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2139	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2140
2141	Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
2142	fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
2143
2144	/* Update CR2 in case it's a page-fault. */
2145	/** @todo This should probably be done much earlier in IEM/PGM. See
2146	* @bugref{5653#c49}. */
2147	if (fFlags & IEM_XCPT_FLAGS_CR2)
2148	pVCpu->cpum.GstCtx.cr2 = uCr2;
2149
2150	/*
2151	* Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
2152	* subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
2153	*/
2154	uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow \| (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
2155	uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
2156	if (uNewTSSLimit < uNewTSSLimitMin)
2157	{
2158	Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
2159	enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
2160	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2161	}
2162
2163	/*
2164	* Task switches in VMX non-root mode always cause task switches.
2165	* The new TSS must have been read and validated (DPL, limits etc.) before a
2166	* task-switch VM-exit commences.
2167	*
2168	* See Intel spec. 25.4.2 "Treatment of Task Switches".
2169	*/
2170	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
2171	{
2172	Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
2173	IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
2174	}
2175
2176	/*
2177	* The SVM nested-guest intercept for task-switch takes priority over all exceptions
2178	* after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
2179	*/
2180	if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
2181	{
2182	uint32_t const uExitInfo1 = SelTSS;
2183	uint32_t uExitInfo2 = uErr;
2184	switch (enmTaskSwitch)
2185	{
2186	case IEMTASKSWITCH_JUMP: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_JUMP; break;
2187	case IEMTASKSWITCH_IRET: uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_IRET; break;
2188	default: break;
2189	}
2190	if (fFlags & IEM_XCPT_FLAGS_ERR)
2191	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
2192	if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
2193	uExitInfo2 \|= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
2194
2195	Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
2196	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
2197	RT_NOREF2(uExitInfo1, uExitInfo2);
2198	}
2199
2200	/*
2201	* Check the current TSS limit. The last written byte to the current TSS during the
2202	* task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
2203	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2204	*
2205	* The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
2206	* end up with smaller than "legal" TSS limits.
2207	*/
2208	uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
2209	uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
2210	if (uCurTSSLimit < uCurTSSLimitMin)
2211	{
2212	Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
2213	enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
2214	return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
2215	}
2216
2217	/*
2218	* Verify that the new TSS can be accessed and map it. Map only the required contents
2219	* and not the entire TSS.
2220	*/
2221	void *pvNewTSS;
2222	uint32_t const cbNewTSS = uNewTSSLimitMin + 1;
2223	RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
2224	AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
2225	/** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
2226	* not perform correct translation if this happens. See Intel spec. 7.2.1
2227	* "Task-State Segment". */
2228	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW, 0);
2229	if (rcStrict != VINF_SUCCESS)
2230	{
2231	Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
2232	cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
2233	return rcStrict;
2234	}
2235
2236	/*
2237	* Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
2238	*/
2239	uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
2240	if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
2241	\|\| enmTaskSwitch == IEMTASKSWITCH_IRET)
2242	{
2243	PX86DESC pDescCurTSS;
2244	rcStrict = iemMemMap(pVCpu, (void *)&pDescCurTSS, sizeof(pDescCurTSS), UINT8_MAX,
2245	pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2246	if (rcStrict != VINF_SUCCESS)
2247	{
2248	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2249	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2250	return rcStrict;
2251	}
2252
2253	pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2254	rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
2255	if (rcStrict != VINF_SUCCESS)
2256	{
2257	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2258	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2259	return rcStrict;
2260	}
2261
2262	/* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
2263	if (enmTaskSwitch == IEMTASKSWITCH_IRET)
2264	{
2265	Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
2266	\|\| uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
2267	u32EFlags &= ~X86_EFL_NT;
2268	}
2269	}
2270
2271	/*
2272	* Save the CPU state into the current TSS.
2273	*/
2274	RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
2275	if (GCPtrNewTSS == GCPtrCurTSS)
2276	{
2277	Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur\|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
2278	Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
2279	pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
2280	pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
2281	pVCpu->cpum.GstCtx.ldtr.Sel));
2282	}
2283	if (fIsNewTSS386)
2284	{
2285	/*
2286	* Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
2287	* See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
2288	*/
2289	void *pvCurTSS32;
2290	uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
2291	uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
2292	AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
2293	rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2294	if (rcStrict != VINF_SUCCESS)
2295	{
2296	Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2297	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2298	return rcStrict;
2299	}
2300
2301	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2302	PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
2303	pCurTSS32->eip = uNextEip;
2304	pCurTSS32->eflags = u32EFlags;
2305	pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
2306	pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
2307	pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
2308	pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
2309	pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
2310	pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
2311	pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
2312	pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
2313	pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
2314	pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
2315	pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
2316	pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
2317	pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
2318	pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
2319
2320	rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
2321	if (rcStrict != VINF_SUCCESS)
2322	{
2323	Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2324	VBOXSTRICTRC_VAL(rcStrict)));
2325	return rcStrict;
2326	}
2327	}
2328	else
2329	{
2330	/*
2331	* Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
2332	*/
2333	void *pvCurTSS16;
2334	uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
2335	uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
2336	AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
2337	rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
2338	if (rcStrict != VINF_SUCCESS)
2339	{
2340	Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
2341	enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
2342	return rcStrict;
2343	}
2344
2345	/* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
2346	PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
2347	pCurTSS16->ip = uNextEip;
2348	pCurTSS16->flags = u32EFlags;
2349	pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
2350	pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
2351	pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
2352	pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
2353	pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
2354	pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
2355	pCurTSS16->si = pVCpu->cpum.GstCtx.si;
2356	pCurTSS16->di = pVCpu->cpum.GstCtx.di;
2357	pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
2358	pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
2359	pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
2360	pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
2361
2362	rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
2363	if (rcStrict != VINF_SUCCESS)
2364	{
2365	Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
2366	VBOXSTRICTRC_VAL(rcStrict)));
2367	return rcStrict;
2368	}
2369	}
2370
2371	/*
2372	* Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
2373	*/
2374	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2375	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2376	{
2377	/* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
2378	PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
2379	pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
2380	}
2381
2382	/*
2383	* Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
2384	* it's done further below with error handling (e.g. CR3 changes will go through PGM).
2385	*/
2386	uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
2387	uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
2388	bool fNewDebugTrap;
2389	if (fIsNewTSS386)
2390	{
2391	PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
2392	uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
2393	uNewEip = pNewTSS32->eip;
2394	uNewEflags = pNewTSS32->eflags;
2395	uNewEax = pNewTSS32->eax;
2396	uNewEcx = pNewTSS32->ecx;
2397	uNewEdx = pNewTSS32->edx;
2398	uNewEbx = pNewTSS32->ebx;
2399	uNewEsp = pNewTSS32->esp;
2400	uNewEbp = pNewTSS32->ebp;
2401	uNewEsi = pNewTSS32->esi;
2402	uNewEdi = pNewTSS32->edi;
2403	uNewES = pNewTSS32->es;
2404	uNewCS = pNewTSS32->cs;
2405	uNewSS = pNewTSS32->ss;
2406	uNewDS = pNewTSS32->ds;
2407	uNewFS = pNewTSS32->fs;
2408	uNewGS = pNewTSS32->gs;
2409	uNewLdt = pNewTSS32->selLdt;
2410	fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
2411	}
2412	else
2413	{
2414	PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
2415	uNewCr3 = 0;
2416	uNewEip = pNewTSS16->ip;
2417	uNewEflags = pNewTSS16->flags;
2418	uNewEax = UINT32_C(0xffff0000) \| pNewTSS16->ax;
2419	uNewEcx = UINT32_C(0xffff0000) \| pNewTSS16->cx;
2420	uNewEdx = UINT32_C(0xffff0000) \| pNewTSS16->dx;
2421	uNewEbx = UINT32_C(0xffff0000) \| pNewTSS16->bx;
2422	uNewEsp = UINT32_C(0xffff0000) \| pNewTSS16->sp;
2423	uNewEbp = UINT32_C(0xffff0000) \| pNewTSS16->bp;
2424	uNewEsi = UINT32_C(0xffff0000) \| pNewTSS16->si;
2425	uNewEdi = UINT32_C(0xffff0000) \| pNewTSS16->di;
2426	uNewES = pNewTSS16->es;
2427	uNewCS = pNewTSS16->cs;
2428	uNewSS = pNewTSS16->ss;
2429	uNewDS = pNewTSS16->ds;
2430	uNewFS = 0;
2431	uNewGS = 0;
2432	uNewLdt = pNewTSS16->selLdt;
2433	fNewDebugTrap = false;
2434	}
2435
2436	if (GCPtrNewTSS == GCPtrCurTSS)
2437	Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
2438	uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
2439
2440	/*
2441	* We're done accessing the new TSS.
2442	*/
2443	rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
2444	if (rcStrict != VINF_SUCCESS)
2445	{
2446	Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
2447	return rcStrict;
2448	}
2449
2450	/*
2451	* Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
2452	*/
2453	if (enmTaskSwitch != IEMTASKSWITCH_IRET)
2454	{
2455	rcStrict = iemMemMap(pVCpu, (void *)&pNewDescTSS, sizeof(pNewDescTSS), UINT8_MAX,
2456	pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
2457	if (rcStrict != VINF_SUCCESS)
2458	{
2459	Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2460	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2461	return rcStrict;
2462	}
2463
2464	/* Check that the descriptor indicates the new TSS is available (not busy). */
2465	AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
2466	\|\| pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
2467	("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
2468
2469	pNewDescTSS->Legacy.Gate.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2470	rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
2471	if (rcStrict != VINF_SUCCESS)
2472	{
2473	Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
2474	enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
2475	return rcStrict;
2476	}
2477	}
2478
2479	/*
2480	* From this point on, we're technically in the new task. We will defer exceptions
2481	* until the completion of the task switch but before executing any instructions in the new task.
2482	*/
2483	pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
2484	pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
2485	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
2486	pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
2487	pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
2488	pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
2489	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
2490
2491	/* Set the busy bit in TR. */
2492	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2493
2494	/* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
2495	if ( enmTaskSwitch == IEMTASKSWITCH_CALL
2496	\|\| enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
2497	{
2498	uNewEflags \|= X86_EFL_NT;
2499	}
2500
2501	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
2502	pVCpu->cpum.GstCtx.cr0 \|= X86_CR0_TS;
2503	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
2504
2505	pVCpu->cpum.GstCtx.eip = uNewEip;
2506	pVCpu->cpum.GstCtx.eax = uNewEax;
2507	pVCpu->cpum.GstCtx.ecx = uNewEcx;
2508	pVCpu->cpum.GstCtx.edx = uNewEdx;
2509	pVCpu->cpum.GstCtx.ebx = uNewEbx;
2510	pVCpu->cpum.GstCtx.esp = uNewEsp;
2511	pVCpu->cpum.GstCtx.ebp = uNewEbp;
2512	pVCpu->cpum.GstCtx.esi = uNewEsi;
2513	pVCpu->cpum.GstCtx.edi = uNewEdi;
2514
2515	uNewEflags &= X86_EFL_LIVE_MASK;
2516	uNewEflags \|= X86_EFL_RA1_MASK;
2517	IEMMISC_SET_EFL(pVCpu, uNewEflags);
2518
2519	/*
2520	* Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
2521	* will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
2522	* due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
2523	*/
2524	pVCpu->cpum.GstCtx.es.Sel = uNewES;
2525	pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
2526
2527	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2528	pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
2529
2530	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2531	pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
2532
2533	pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
2534	pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
2535
2536	pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
2537	pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
2538
2539	pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
2540	pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
2541	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
2542
2543	pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
2544	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
2545	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
2546	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
2547
2548	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2549	{
2550	pVCpu->cpum.GstCtx.es.Attr.u \|= X86DESCATTR_UNUSABLE;
2551	pVCpu->cpum.GstCtx.cs.Attr.u \|= X86DESCATTR_UNUSABLE;
2552	pVCpu->cpum.GstCtx.ss.Attr.u \|= X86DESCATTR_UNUSABLE;
2553	pVCpu->cpum.GstCtx.ds.Attr.u \|= X86DESCATTR_UNUSABLE;
2554	pVCpu->cpum.GstCtx.fs.Attr.u \|= X86DESCATTR_UNUSABLE;
2555	pVCpu->cpum.GstCtx.gs.Attr.u \|= X86DESCATTR_UNUSABLE;
2556	pVCpu->cpum.GstCtx.ldtr.Attr.u \|= X86DESCATTR_UNUSABLE;
2557	}
2558
2559	/*
2560	* Switch CR3 for the new task.
2561	*/
2562	if ( fIsNewTSS386
2563	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
2564	{
2565	/** @todo Should we update and flush TLBs only if CR3 value actually changes? */
2566	int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
2567	AssertRCSuccessReturn(rc, rc);
2568
2569	/* Inform PGM. */
2570	/** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
2571	rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
2572	AssertRCReturn(rc, rc);
2573	/* ignore informational status codes */
2574
2575	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
2576	}
2577
2578	/*
2579	* Switch LDTR for the new task.
2580	*/
2581	if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
2582	iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
2583	else
2584	{
2585	Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
2586
2587	IEMSELDESC DescNewLdt;
2588	rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
2589	if (rcStrict != VINF_SUCCESS)
2590	{
2591	Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
2592	uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
2593	return rcStrict;
2594	}
2595	if ( !DescNewLdt.Legacy.Gen.u1Present
2596	\|\| DescNewLdt.Legacy.Gen.u1DescType
2597	\|\| DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
2598	{
2599	Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
2600	uNewLdt, DescNewLdt.Legacy.u));
2601	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
2602	}
2603
2604	pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
2605	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
2606	pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
2607	pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
2608	pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
2609	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
2610	pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
2611	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
2612	}
2613
2614	IEMSELDESC DescSS;
2615	if (IEM_IS_V86_MODE(pVCpu))
2616	{
2617	pVCpu->iem.s.uCpl = 3;
2618	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
2619	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
2620	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
2621	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
2622	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
2623	iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
2624
2625	/* Quick fix: fake DescSS. / /* @todo fix the code further down? */
2626	DescSS.Legacy.u = 0;
2627	DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
2628	DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
2629	DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
2630	DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
2631	DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
2632	DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2633	DescSS.Legacy.Gen.u2Dpl = 3;
2634	}
2635	else
2636	{
2637	uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
2638
2639	/*
2640	* Load the stack segment for the new task.
2641	*/
2642	if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
2643	{
2644	Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
2645	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2646	}
2647
2648	/* Fetch the descriptor. */
2649	rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
2650	if (rcStrict != VINF_SUCCESS)
2651	{
2652	Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
2653	VBOXSTRICTRC_VAL(rcStrict)));
2654	return rcStrict;
2655	}
2656
2657	/* SS must be a data segment and writable. */
2658	if ( !DescSS.Legacy.Gen.u1DescType
2659	\|\| (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
2660	\|\| !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
2661	{
2662	Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
2663	uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
2664	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2665	}
2666
2667	/* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
2668	if ( (uNewSS & X86_SEL_RPL) != uNewCpl
2669	\|\| DescSS.Legacy.Gen.u2Dpl != uNewCpl)
2670	{
2671	Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
2672	uNewCpl));
2673	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2674	}
2675
2676	/* Is it there? */
2677	if (!DescSS.Legacy.Gen.u1Present)
2678	{
2679	Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
2680	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
2681	}
2682
2683	uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
2684	uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
2685
2686	/* Set the accessed bit before committing the result into SS. */
2687	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2688	{
2689	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
2690	if (rcStrict != VINF_SUCCESS)
2691	return rcStrict;
2692	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2693	}
2694
2695	/* Commit SS. */
2696	pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
2697	pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
2698	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
2699	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
2700	pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
2701	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
2702	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
2703
2704	/* CPL has changed, update IEM before loading rest of segments. */
2705	pVCpu->iem.s.uCpl = uNewCpl;
2706
2707	/*
2708	* Load the data segments for the new task.
2709	*/
2710	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
2711	if (rcStrict != VINF_SUCCESS)
2712	return rcStrict;
2713	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
2714	if (rcStrict != VINF_SUCCESS)
2715	return rcStrict;
2716	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
2717	if (rcStrict != VINF_SUCCESS)
2718	return rcStrict;
2719	rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
2720	if (rcStrict != VINF_SUCCESS)
2721	return rcStrict;
2722
2723	/*
2724	* Load the code segment for the new task.
2725	*/
2726	if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
2727	{
2728	Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
2729	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2730	}
2731
2732	/* Fetch the descriptor. */
2733	IEMSELDESC DescCS;
2734	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
2735	if (rcStrict != VINF_SUCCESS)
2736	{
2737	Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
2738	return rcStrict;
2739	}
2740
2741	/* CS must be a code segment. */
2742	if ( !DescCS.Legacy.Gen.u1DescType
2743	\|\| !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
2744	{
2745	Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
2746	DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
2747	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2748	}
2749
2750	/* For conforming CS, DPL must be less than or equal to the RPL. */
2751	if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2752	&& DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
2753	{
2754	Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
2755	DescCS.Legacy.Gen.u2Dpl));
2756	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2757	}
2758
2759	/* For non-conforming CS, DPL must match RPL. */
2760	if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
2761	&& DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
2762	{
2763	Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
2764	DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
2765	return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2766	}
2767
2768	/* Is it there? */
2769	if (!DescCS.Legacy.Gen.u1Present)
2770	{
2771	Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
2772	return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
2773	}
2774
2775	cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
2776	u64Base = X86DESC_BASE(&DescCS.Legacy);
2777
2778	/* Set the accessed bit before committing the result into CS. */
2779	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
2780	{
2781	rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
2782	if (rcStrict != VINF_SUCCESS)
2783	return rcStrict;
2784	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
2785	}
2786
2787	/* Commit CS. */
2788	pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
2789	pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
2790	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
2791	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
2792	pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
2793	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
2794	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
2795	}
2796
2797	/** @todo Debug trap. */
2798	if (fIsNewTSS386 && fNewDebugTrap)
2799	Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
2800
2801	/*
2802	* Construct the error code masks based on what caused this task switch.
2803	* See Intel Instruction reference for INT.
2804	*/
2805	uint16_t uExt;
2806	if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
2807	&& ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2808	\|\| (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
2809	{
2810	uExt = 1;
2811	}
2812	else
2813	uExt = 0;
2814
2815	/*
2816	* Push any error code on to the new stack.
2817	*/
2818	if (fFlags & IEM_XCPT_FLAGS_ERR)
2819	{
2820	Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
2821	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
2822	uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
2823
2824	/* Check that there is sufficient space on the stack. */
2825	/** @todo Factor out segment limit checking for normal/expand down segments
2826	* into a separate function. */
2827	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
2828	{
2829	if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
2830	\|\| pVCpu->cpum.GstCtx.esp < cbStackFrame)
2831	{
2832	/** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2833	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
2834	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2835	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2836	}
2837	}
2838	else
2839	{
2840	if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2841	\|\| pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
2842	{
2843	Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
2844	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
2845	return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
2846	}
2847	}
2848
2849
2850	if (fIsNewTSS386)
2851	rcStrict = iemMemStackPushU32(pVCpu, uErr);
2852	else
2853	rcStrict = iemMemStackPushU16(pVCpu, uErr);
2854	if (rcStrict != VINF_SUCCESS)
2855	{
2856	Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
2857	fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
2858	return rcStrict;
2859	}
2860	}
2861
2862	/* Check the new EIP against the new CS limit. */
2863	if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
2864	{
2865	Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
2866	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
2867	/** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
2868	return iemRaiseGeneralProtectionFault(pVCpu, uExt);
2869	}
2870
2871	Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
2872	pVCpu->cpum.GstCtx.ss.Sel));
2873	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
2874	}
2875
2876
2877	/**
2878	* Implements exceptions and interrupts for protected mode.
2879	*
2880	* @returns VBox strict status code.
2881	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2882	* @param cbInstr The number of bytes to offset rIP by in the return
2883	* address.
2884	* @param u8Vector The interrupt / exception vector number.
2885	* @param fFlags The flags.
2886	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
2887	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
2888	*/
2889	static VBOXSTRICTRC
2890	iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
2891	uint8_t cbInstr,
2892	uint8_t u8Vector,
2893	uint32_t fFlags,
2894	uint16_t uErr,
2895	uint64_t uCr2) RT_NOEXCEPT
2896	{
2897	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
2898
2899	/*
2900	* Read the IDT entry.
2901	*/
2902	if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
2903	{
2904	Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
2905	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2906	}
2907	X86DESC Idte;
2908	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
2909	pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
2910	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
2911	{
2912	Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
2913	return rcStrict;
2914	}
2915	Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
2916	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
2917	Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
2918
2919	/*
2920	* Check the descriptor type, DPL and such.
2921	* ASSUMES this is done in the same order as described for call-gate calls.
2922	*/
2923	if (Idte.Gate.u1DescType)
2924	{
2925	Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2926	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2927	}
2928	bool fTaskGate = false;
2929	uint8_t f32BitGate = true;
2930	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
2931	switch (Idte.Gate.u4Type)
2932	{
2933	case X86_SEL_TYPE_SYS_UNDEFINED:
2934	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
2935	case X86_SEL_TYPE_SYS_LDT:
2936	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
2937	case X86_SEL_TYPE_SYS_286_CALL_GATE:
2938	case X86_SEL_TYPE_SYS_UNDEFINED2:
2939	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
2940	case X86_SEL_TYPE_SYS_UNDEFINED3:
2941	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
2942	case X86_SEL_TYPE_SYS_386_CALL_GATE:
2943	case X86_SEL_TYPE_SYS_UNDEFINED4:
2944	{
2945	/** @todo check what actually happens when the type is wrong...
2946	* esp. call gates. */
2947	Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
2948	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2949	}
2950
2951	case X86_SEL_TYPE_SYS_286_INT_GATE:
2952	f32BitGate = false;
2953	RT_FALL_THRU();
2954	case X86_SEL_TYPE_SYS_386_INT_GATE:
2955	fEflToClear \|= X86_EFL_IF;
2956	break;
2957
2958	case X86_SEL_TYPE_SYS_TASK_GATE:
2959	fTaskGate = true;
2960	#ifndef IEM_IMPLEMENTS_TASKSWITCH
2961	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
2962	#endif
2963	break;
2964
2965	case X86_SEL_TYPE_SYS_286_TRAP_GATE:
2966	f32BitGate = false;
2967	case X86_SEL_TYPE_SYS_386_TRAP_GATE:
2968	break;
2969
2970	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2971	}
2972
2973	/* Check DPL against CPL if applicable. */
2974	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
2975	{
2976	if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
2977	{
2978	Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
2979	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2980	}
2981	}
2982
2983	/* Is it there? */
2984	if (!Idte.Gate.u1Present)
2985	{
2986	Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
2987	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
2988	}
2989
2990	/* Is it a task-gate? */
2991	if (fTaskGate)
2992	{
2993	/*
2994	* Construct the error code masks based on what caused this task switch.
2995	* See Intel Instruction reference for INT.
2996	*/
2997	uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
2998	&& !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
2999	uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
3000	RTSEL SelTSS = Idte.Gate.u16Sel;
3001
3002	/*
3003	* Fetch the TSS descriptor in the GDT.
3004	*/
3005	IEMSELDESC DescTSS;
3006	rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) \| uExt);
3007	if (rcStrict != VINF_SUCCESS)
3008	{
3009	Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
3010	VBOXSTRICTRC_VAL(rcStrict)));
3011	return rcStrict;
3012	}
3013
3014	/* The TSS descriptor must be a system segment and be available (not busy). */
3015	if ( DescTSS.Legacy.Gen.u1DescType
3016	\|\| ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
3017	&& DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
3018	{
3019	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
3020	u8Vector, SelTSS, DescTSS.Legacy.au64));
3021	return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) \| uExt);
3022	}
3023
3024	/* The TSS must be present. */
3025	if (!DescTSS.Legacy.Gen.u1Present)
3026	{
3027	Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
3028	return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) \| uExt);
3029	}
3030
3031	/* Do the actual task switch. */
3032	return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
3033	(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
3034	fFlags, uErr, uCr2, SelTSS, &DescTSS);
3035	}
3036
3037	/* A null CS is bad. */
3038	RTSEL NewCS = Idte.Gate.u16Sel;
3039	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3040	{
3041	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3042	return iemRaiseGeneralProtectionFault0(pVCpu);
3043	}
3044
3045	/* Fetch the descriptor for the new CS. */
3046	IEMSELDESC DescCS;
3047	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
3048	if (rcStrict != VINF_SUCCESS)
3049	{
3050	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3051	return rcStrict;
3052	}
3053
3054	/* Must be a code segment. */
3055	if (!DescCS.Legacy.Gen.u1DescType)
3056	{
3057	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3058	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3059	}
3060	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
3061	{
3062	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3063	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3064	}
3065
3066	/* Don't allow lowering the privilege level. */
3067	/** @todo Does the lowering of privileges apply to software interrupts
3068	* only? This has bearings on the more-privileged or
3069	* same-privilege stack behavior further down. A testcase would
3070	* be nice. */
3071	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3072	{
3073	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3074	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3075	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3076	}
3077
3078	/* Make sure the selector is present. */
3079	if (!DescCS.Legacy.Gen.u1Present)
3080	{
3081	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3082	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3083	}
3084
3085	/* Check the new EIP against the new CS limit. */
3086	uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
3087	\|\| Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
3088	? Idte.Gate.u16OffsetLow
3089	: Idte.Gate.u16OffsetLow \| ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
3090	uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
3091	if (uNewEip > cbLimitCS)
3092	{
3093	Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
3094	u8Vector, uNewEip, cbLimitCS, NewCS));
3095	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3096	}
3097	Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
3098
3099	/* Calc the flag image to push. */
3100	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3101	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3102	fEfl &= ~X86_EFL_RF;
3103	else
3104	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3105
3106	/* From V8086 mode only go to CPL 0. */
3107	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3108	? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3109	if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
3110	{
3111	Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
3112	return iemRaiseGeneralProtectionFault(pVCpu, 0);
3113	}
3114
3115	/*
3116	* If the privilege level changes, we need to get a new stack from the TSS.
3117	* This in turns means validating the new SS and ESP...
3118	*/
3119	if (uNewCpl != pVCpu->iem.s.uCpl)
3120	{
3121	RTSEL NewSS;
3122	uint32_t uNewEsp;
3123	rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
3124	if (rcStrict != VINF_SUCCESS)
3125	return rcStrict;
3126
3127	IEMSELDESC DescSS;
3128	rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
3129	if (rcStrict != VINF_SUCCESS)
3130	return rcStrict;
3131	/* If the new SS is 16-bit, we are only going to use SP, not ESP. */
3132	if (!DescSS.Legacy.Gen.u1DefBig)
3133	{
3134	Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
3135	uNewEsp = (uint16_t)uNewEsp;
3136	}
3137
3138	Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3139
3140	/* Check that there is sufficient space for the stack frame. */
3141	uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
3142	uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
3143	? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
3144	: (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
3145
3146	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
3147	{
3148	if ( uNewEsp - 1 > cbLimitSS
3149	\|\| uNewEsp < cbStackFrame)
3150	{
3151	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
3152	u8Vector, NewSS, uNewEsp, cbStackFrame));
3153	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3154	}
3155	}
3156	else
3157	{
3158	if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
3159	\|\| uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
3160	{
3161	Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
3162	u8Vector, NewSS, uNewEsp, cbStackFrame));
3163	return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
3164	}
3165	}
3166
3167	/*
3168	* Start making changes.
3169	*/
3170
3171	/* Set the new CPL so that stack accesses use it. */
3172	uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3173	pVCpu->iem.s.uCpl = uNewCpl;
3174
3175	/* Create the stack frame. */
3176	RTPTRUNION uStackFrame;
3177	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3178	uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy),
3179	IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3180	if (rcStrict != VINF_SUCCESS)
3181	return rcStrict;
3182	void * const pvStackFrame = uStackFrame.pv;
3183	if (f32BitGate)
3184	{
3185	if (fFlags & IEM_XCPT_FLAGS_ERR)
3186	*uStackFrame.pu32++ = uErr;
3187	uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3188	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3189	uStackFrame.pu32[2] = fEfl;
3190	uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
3191	uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
3192	Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
3193	if (fEfl & X86_EFL_VM)
3194	{
3195	uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
3196	uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
3197	uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
3198	uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
3199	uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
3200	}
3201	}
3202	else
3203	{
3204	if (fFlags & IEM_XCPT_FLAGS_ERR)
3205	*uStackFrame.pu16++ = uErr;
3206	uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3207	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl;
3208	uStackFrame.pu16[2] = fEfl;
3209	uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
3210	uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
3211	Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
3212	if (fEfl & X86_EFL_VM)
3213	{
3214	uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
3215	uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
3216	uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
3217	uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
3218	uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
3219	}
3220	}
3221	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3222	if (rcStrict != VINF_SUCCESS)
3223	return rcStrict;
3224
3225	/* Mark the selectors 'accessed' (hope this is the correct time). */
3226	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3227	* after pushing the stack frame? (Write protect the gdt + stack to
3228	* find out.) */
3229	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3230	{
3231	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3232	if (rcStrict != VINF_SUCCESS)
3233	return rcStrict;
3234	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3235	}
3236
3237	if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3238	{
3239	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
3240	if (rcStrict != VINF_SUCCESS)
3241	return rcStrict;
3242	DescSS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3243	}
3244
3245	/*
3246	* Start comitting the register changes (joins with the DPL=CPL branch).
3247	*/
3248	pVCpu->cpum.GstCtx.ss.Sel = NewSS;
3249	pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
3250	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3251	pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
3252	pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
3253	pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
3254	/** @todo When coming from 32-bit code and operating with a 16-bit TSS and
3255	* 16-bit handler, the high word of ESP remains unchanged (i.e. only
3256	* SP is loaded).
3257	* Need to check the other combinations too:
3258	* - 16-bit TSS, 32-bit handler
3259	* - 32-bit TSS, 16-bit handler */
3260	if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3261	pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
3262	else
3263	pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
3264
3265	if (fEfl & X86_EFL_VM)
3266	{
3267	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
3268	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
3269	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
3270	iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
3271	}
3272	}
3273	/*
3274	* Same privilege, no stack change and smaller stack frame.
3275	*/
3276	else
3277	{
3278	uint64_t uNewRsp;
3279	RTPTRUNION uStackFrame;
3280	uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
3281	rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1, &uStackFrame.pv, &uNewRsp);
3282	if (rcStrict != VINF_SUCCESS)
3283	return rcStrict;
3284	void * const pvStackFrame = uStackFrame.pv;
3285
3286	if (f32BitGate)
3287	{
3288	if (fFlags & IEM_XCPT_FLAGS_ERR)
3289	*uStackFrame.pu32++ = uErr;
3290	uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3291	uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| pVCpu->iem.s.uCpl;
3292	uStackFrame.pu32[2] = fEfl;
3293	}
3294	else
3295	{
3296	if (fFlags & IEM_XCPT_FLAGS_ERR)
3297	*uStackFrame.pu16++ = uErr;
3298	uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
3299	uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| pVCpu->iem.s.uCpl;
3300	uStackFrame.pu16[2] = fEfl;
3301	}
3302	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
3303	if (rcStrict != VINF_SUCCESS)
3304	return rcStrict;
3305
3306	/* Mark the CS selector as 'accessed'. */
3307	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3308	{
3309	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3310	if (rcStrict != VINF_SUCCESS)
3311	return rcStrict;
3312	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3313	}
3314
3315	/*
3316	* Start committing the register changes (joins with the other branch).
3317	*/
3318	pVCpu->cpum.GstCtx.rsp = uNewRsp;
3319	}
3320
3321	/* ... register committing continues. */
3322	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3323	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3324	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3325	pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
3326	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3327	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3328
3329	pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
3330	fEfl &= ~fEflToClear;
3331	IEMMISC_SET_EFL(pVCpu, fEfl);
3332
3333	if (fFlags & IEM_XCPT_FLAGS_CR2)
3334	pVCpu->cpum.GstCtx.cr2 = uCr2;
3335
3336	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3337	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3338
3339	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3340	}
3341
3342
3343	/**
3344	* Implements exceptions and interrupts for long mode.
3345	*
3346	* @returns VBox strict status code.
3347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3348	* @param cbInstr The number of bytes to offset rIP by in the return
3349	* address.
3350	* @param u8Vector The interrupt / exception vector number.
3351	* @param fFlags The flags.
3352	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3353	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3354	*/
3355	static VBOXSTRICTRC
3356	iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
3357	uint8_t cbInstr,
3358	uint8_t u8Vector,
3359	uint32_t fFlags,
3360	uint16_t uErr,
3361	uint64_t uCr2) RT_NOEXCEPT
3362	{
3363	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3364
3365	/*
3366	* Read the IDT entry.
3367	*/
3368	uint16_t offIdt = (uint16_t)u8Vector << 4;
3369	if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
3370	{
3371	Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3372	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3373	}
3374	X86DESC64 Idte;
3375	#ifdef _MSC_VER /* Shut up silly compiler warning. */
3376	Idte.au64[0] = 0;
3377	Idte.au64[1] = 0;
3378	#endif
3379	VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
3380	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
3381	rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
3382	if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3383	{
3384	Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3385	return rcStrict;
3386	}
3387	Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
3388	u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
3389	Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
3390
3391	/*
3392	* Check the descriptor type, DPL and such.
3393	* ASSUMES this is done in the same order as described for call-gate calls.
3394	*/
3395	if (Idte.Gate.u1DescType)
3396	{
3397	Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3398	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3399	}
3400	uint32_t fEflToClear = X86_EFL_TF \| X86_EFL_NT \| X86_EFL_RF \| X86_EFL_VM;
3401	switch (Idte.Gate.u4Type)
3402	{
3403	case AMD64_SEL_TYPE_SYS_INT_GATE:
3404	fEflToClear \|= X86_EFL_IF;
3405	break;
3406	case AMD64_SEL_TYPE_SYS_TRAP_GATE:
3407	break;
3408
3409	default:
3410	Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
3411	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3412	}
3413
3414	/* Check DPL against CPL if applicable. */
3415	if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
3416	{
3417	if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
3418	{
3419	Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
3420	return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3421	}
3422	}
3423
3424	/* Is it there? */
3425	if (!Idte.Gate.u1Present)
3426	{
3427	Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
3428	return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT \| ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3429	}
3430
3431	/* A null CS is bad. */
3432	RTSEL NewCS = Idte.Gate.u16Sel;
3433	if (!(NewCS & X86_SEL_MASK_OFF_RPL))
3434	{
3435	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
3436	return iemRaiseGeneralProtectionFault0(pVCpu);
3437	}
3438
3439	/* Fetch the descriptor for the new CS. */
3440	IEMSELDESC DescCS;
3441	rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
3442	if (rcStrict != VINF_SUCCESS)
3443	{
3444	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
3445	return rcStrict;
3446	}
3447
3448	/* Must be a 64-bit code segment. */
3449	if (!DescCS.Long.Gen.u1DescType)
3450	{
3451	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
3452	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3453	}
3454	if ( !DescCS.Long.Gen.u1Long
3455	\|\| DescCS.Long.Gen.u1DefBig
3456	\|\| !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
3457	{
3458	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
3459	u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
3460	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3461	}
3462
3463	/* Don't allow lowering the privilege level. For non-conforming CS
3464	selectors, the CS.DPL sets the privilege level the trap/interrupt
3465	handler runs at. For conforming CS selectors, the CPL remains
3466	unchanged, but the CS.DPL must be <= CPL. */
3467	/** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
3468	* when CPU in Ring-0. Result \#GP? */
3469	if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
3470	{
3471	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
3472	u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3473	return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
3474	}
3475
3476
3477	/* Make sure the selector is present. */
3478	if (!DescCS.Legacy.Gen.u1Present)
3479	{
3480	Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
3481	return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
3482	}
3483
3484	/* Check that the new RIP is canonical. */
3485	uint64_t const uNewRip = Idte.Gate.u16OffsetLow
3486	\| ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
3487	\| ((uint64_t)Idte.Gate.u32OffsetTop << 32);
3488	if (!IEM_IS_CANONICAL(uNewRip))
3489	{
3490	Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
3491	return iemRaiseGeneralProtectionFault0(pVCpu);
3492	}
3493
3494	/*
3495	* If the privilege level changes or if the IST isn't zero, we need to get
3496	* a new stack from the TSS.
3497	*/
3498	uint64_t uNewRsp;
3499	uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
3500	? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
3501	if ( uNewCpl != pVCpu->iem.s.uCpl
3502	\|\| Idte.Gate.u3IST != 0)
3503	{
3504	rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
3505	if (rcStrict != VINF_SUCCESS)
3506	return rcStrict;
3507	}
3508	else
3509	uNewRsp = pVCpu->cpum.GstCtx.rsp;
3510	uNewRsp &= ~(uint64_t)0xf;
3511
3512	/*
3513	* Calc the flag image to push.
3514	*/
3515	uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3516	if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP \| IEM_XCPT_FLAGS_T_SOFT_INT))
3517	fEfl &= ~X86_EFL_RF;
3518	else
3519	fEfl \|= X86_EFL_RF; /* Vagueness is all I've found on this so far... / /* @todo Automatically pushing EFLAGS.RF. */
3520
3521	/*
3522	* Start making changes.
3523	*/
3524	/* Set the new CPL so that stack accesses use it. */
3525	uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
3526	pVCpu->iem.s.uCpl = uNewCpl;
3527
3528	/* Create the stack frame. */
3529	uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
3530	RTPTRUNION uStackFrame;
3531	rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
3532	uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
3533	if (rcStrict != VINF_SUCCESS)
3534	return rcStrict;
3535	void * const pvStackFrame = uStackFrame.pv;
3536
3537	if (fFlags & IEM_XCPT_FLAGS_ERR)
3538	*uStackFrame.pu64++ = uErr;
3539	uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
3540	uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) \| uOldCpl; /* CPL paranoia */
3541	uStackFrame.pu64[2] = fEfl;
3542	uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
3543	uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
3544	rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W \| IEM_ACCESS_WHAT_SYS);
3545	if (rcStrict != VINF_SUCCESS)
3546	return rcStrict;
3547
3548	/* Mark the CS selectors 'accessed' (hope this is the correct time). */
3549	/** @todo testcase: excatly _when_ are the accessed bits set - before or
3550	* after pushing the stack frame? (Write protect the gdt + stack to
3551	* find out.) */
3552	if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3553	{
3554	rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
3555	if (rcStrict != VINF_SUCCESS)
3556	return rcStrict;
3557	DescCS.Legacy.Gen.u4Type \|= X86_SEL_TYPE_ACCESSED;
3558	}
3559
3560	/*
3561	* Start comitting the register changes.
3562	*/
3563	/** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
3564	* hidden registers when interrupting 32-bit or 16-bit code! */
3565	if (uNewCpl != uOldCpl)
3566	{
3567	pVCpu->cpum.GstCtx.ss.Sel = 0 \| uNewCpl;
3568	pVCpu->cpum.GstCtx.ss.ValidSel = 0 \| uNewCpl;
3569	pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
3570	pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
3571	pVCpu->cpum.GstCtx.ss.u64Base = 0;
3572	pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) \| X86DESCATTR_UNUSABLE;
3573	}
3574	pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
3575	pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3576	pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) \| uNewCpl;
3577	pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3578	pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
3579	pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
3580	pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
3581	pVCpu->cpum.GstCtx.rip = uNewRip;
3582
3583	fEfl &= ~fEflToClear;
3584	IEMMISC_SET_EFL(pVCpu, fEfl);
3585
3586	if (fFlags & IEM_XCPT_FLAGS_CR2)
3587	pVCpu->cpum.GstCtx.cr2 = uCr2;
3588
3589	if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3590	iemRaiseXcptAdjustState(pVCpu, u8Vector);
3591
3592	return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3593	}
3594
3595
3596	/**
3597	* Implements exceptions and interrupts.
3598	*
3599	* All exceptions and interrupts goes thru this function!
3600	*
3601	* @returns VBox strict status code.
3602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3603	* @param cbInstr The number of bytes to offset rIP by in the return
3604	* address.
3605	* @param u8Vector The interrupt / exception vector number.
3606	* @param fFlags The flags.
3607	* @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3608	* @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3609	*/
3610	VBOXSTRICTRC
3611	iemRaiseXcptOrInt(PVMCPUCC pVCpu,
3612	uint8_t cbInstr,
3613	uint8_t u8Vector,
3614	uint32_t fFlags,
3615	uint16_t uErr,
3616	uint64_t uCr2) RT_NOEXCEPT
3617	{
3618	/*
3619	* Get all the state that we might need here.
3620	*/
3621	IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3622	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3623
3624	#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
3625	/*
3626	* Flush prefetch buffer
3627	*/
3628	pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
3629	#endif
3630
3631	/*
3632	* Perform the V8086 IOPL check and upgrade the fault without nesting.
3633	*/
3634	if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
3635	&& pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
3636	&& (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
3637	\| IEM_XCPT_FLAGS_BP_INSTR
3638	\| IEM_XCPT_FLAGS_ICEBP_INSTR
3639	\| IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
3640	&& (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
3641	{
3642	Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
3643	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3644	u8Vector = X86_XCPT_GP;
3645	uErr = 0;
3646	}
3647	#ifdef DBGFTRACE_ENABLED
3648	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
3649	pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
3650	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
3651	#endif
3652
3653	/*
3654	* Evaluate whether NMI blocking should be in effect.
3655	* Normally, NMI blocking is in effect whenever we inject an NMI.
3656	*/
3657	bool fBlockNmi;
3658	if ( u8Vector == X86_XCPT_NMI
3659	&& (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3660	fBlockNmi = true;
3661	else
3662	fBlockNmi = false;
3663
3664	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3665	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3666	{
3667	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
3668	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3669	return rcStrict0;
3670
3671	/* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
3672	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
3673	{
3674	Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
3675	fBlockNmi = false;
3676	}
3677	}
3678	#endif
3679
3680	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3681	if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
3682	{
3683	/*
3684	* If the event is being injected as part of VMRUN, it isn't subject to event
3685	* intercepts in the nested-guest. However, secondary exceptions that occur
3686	* during injection of any event -are- subject to exception intercepts.
3687	*
3688	* See AMD spec. 15.20 "Event Injection".
3689	*/
3690	if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
3691	pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
3692	else
3693	{
3694	/*
3695	* Check and handle if the event being raised is intercepted.
3696	*/
3697	VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
3698	if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
3699	return rcStrict0;
3700	}
3701	}
3702	#endif
3703
3704	/*
3705	* Set NMI blocking if necessary.
3706	*/
3707	if ( fBlockNmi
3708	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3709	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3710
3711	/*
3712	* Do recursion accounting.
3713	*/
3714	uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
3715	uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
3716	if (pVCpu->iem.s.cXcptRecursions == 0)
3717	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
3718	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
3719	else
3720	{
3721	Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
3722	u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
3723	pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
3724
3725	if (pVCpu->iem.s.cXcptRecursions >= 4)
3726	{
3727	#ifdef DEBUG_bird
3728	AssertFailed();
3729	#endif
3730	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
3731	}
3732
3733	/*
3734	* Evaluate the sequence of recurring events.
3735	*/
3736	IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
3737	NULL /* pXcptRaiseInfo */);
3738	if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
3739	{ /* likely */ }
3740	else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
3741	{
3742	Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
3743	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR;
3744	u8Vector = X86_XCPT_DF;
3745	uErr = 0;
3746	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3747	/* VMX nested-guest #DF intercept needs to be checked here. */
3748	if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3749	{
3750	VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
3751	if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
3752	return rcStrict0;
3753	}
3754	#endif
3755	/* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
3756	if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
3757	IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 /, 0 / uExitInfo2 */);
3758	}
3759	else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
3760	{
3761	Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
3762	return iemInitiateCpuShutdown(pVCpu);
3763	}
3764	else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
3765	{
3766	/* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
3767	Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
3768	if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
3769	&& !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
3770	return VERR_EM_GUEST_CPU_HANG;
3771	}
3772	else
3773	{
3774	AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
3775	enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
3776	return VERR_IEM_IPE_9;
3777	}
3778
3779	/*
3780	* The 'EXT' bit is set when an exception occurs during deliver of an external
3781	* event (such as an interrupt or earlier exception)[1]. Privileged software
3782	* exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
3783	* interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
3784	*
3785	* [1] - Intel spec. 6.13 "Error Code"
3786	* [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
3787	* [3] - Intel Instruction reference for INT n.
3788	*/
3789	if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_T_EXT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR))
3790	&& (fFlags & IEM_XCPT_FLAGS_ERR)
3791	&& u8Vector != X86_XCPT_PF
3792	&& u8Vector != X86_XCPT_DF)
3793	{
3794	uErr \|= X86_TRAP_ERR_EXTERNAL;
3795	}
3796	}
3797
3798	pVCpu->iem.s.cXcptRecursions++;
3799	pVCpu->iem.s.uCurXcpt = u8Vector;
3800	pVCpu->iem.s.fCurXcpt = fFlags;
3801	pVCpu->iem.s.uCurXcptErr = uErr;
3802	pVCpu->iem.s.uCurXcptCr2 = uCr2;
3803
3804	/*
3805	* Extensive logging.
3806	*/
3807	#if defined(LOG_ENABLED) && defined(IN_RING3)
3808	if (LogIs3Enabled())
3809	{
3810	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
3811	PVM pVM = pVCpu->CTX_SUFF(pVM);
3812	char szRegs[4096];
3813	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
3814	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
3815	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
3816	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
3817	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
3818	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
3819	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
3820	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
3821	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
3822	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
3823	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
3824	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
3825	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
3826	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
3827	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
3828	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
3829	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
3830	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
3831	" efer=%016VR{efer}\n"
3832	" pat=%016VR{pat}\n"
3833	" sf_mask=%016VR{sf_mask}\n"
3834	"krnl_gs_base=%016VR{krnl_gs_base}\n"
3835	" lstar=%016VR{lstar}\n"
3836	" star=%016VR{star} cstar=%016VR{cstar}\n"
3837	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
3838	);
3839
3840	char szInstr[256];
3841	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
3842	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
3843	szInstr, sizeof(szInstr), NULL);
3844	Log3(("%s%s\n", szRegs, szInstr));
3845	}
3846	#endif /* LOG_ENABLED */
3847
3848	/*
3849	* Stats.
3850	*/
3851	if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
3852	STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; });
3853	else if (u8Vector <= X86_XCPT_LAST)
3854	{
3855	STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]);
3856	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector),
3857	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, ASMReadTSC());
3858	}
3859
3860	/*
3861	* #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3)
3862	* to ensure that a stale TLB or paging cache entry will only cause one
3863	* spurious #PF.
3864	*/
3865	if ( u8Vector == X86_XCPT_PF
3866	&& (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_CR2))
3867	IEMTlbInvalidatePage(pVCpu, uCr2);
3868
3869	/*
3870	* Call the mode specific worker function.
3871	*/
3872	VBOXSTRICTRC rcStrict;
3873	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
3874	rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3875	else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
3876	rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3877	else
3878	rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3879
3880	/* Flush the prefetch buffer. */
3881	#ifdef IEM_WITH_CODE_TLB
3882	pVCpu->iem.s.pbInstrBuf = NULL;
3883	#else
3884	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
3885	#endif
3886
3887	/*
3888	* Unwind.
3889	*/
3890	pVCpu->iem.s.cXcptRecursions--;
3891	pVCpu->iem.s.uCurXcpt = uPrevXcpt;
3892	pVCpu->iem.s.fCurXcpt = fPrevXcpt;
3893	Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
3894	VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
3895	pVCpu->iem.s.cXcptRecursions + 1));
3896	return rcStrict;
3897	}
3898
3899	#ifdef IEM_WITH_SETJMP
3900	/**
3901	* See iemRaiseXcptOrInt. Will not return.
3902	*/
3903	DECL_NO_RETURN(void)
3904	iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
3905	uint8_t cbInstr,
3906	uint8_t u8Vector,
3907	uint32_t fFlags,
3908	uint16_t uErr,
3909	uint64_t uCr2) RT_NOEXCEPT
3910	{
3911	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
3912	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3913	}
3914	#endif
3915
3916
3917	/** \#DE - 00. */
3918	VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT
3919	{
3920	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3921	}
3922
3923
3924	/** \#DB - 01.
3925	* @note This automatically clear DR7.GD. */
3926	VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT
3927	{
3928	/** @todo set/clear RF. */
3929	pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3930	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3931	}
3932
3933
3934	/** \#BR - 05. */
3935	VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT
3936	{
3937	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3938	}
3939
3940
3941	/** \#UD - 06. */
3942	VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT
3943	{
3944	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3945	}
3946
3947
3948	/** \#NM - 07. */
3949	VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT
3950	{
3951	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
3952	}
3953
3954
3955	/** \#TS(err) - 0a. */
3956	VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3957	{
3958	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
3959	}
3960
3961
3962	/** \#TS(tr) - 0a. */
3963	VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT
3964	{
3965	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3966	pVCpu->cpum.GstCtx.tr.Sel, 0);
3967	}
3968
3969
3970	/** \#TS(0) - 0a. */
3971	VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
3972	{
3973	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3974	0, 0);
3975	}
3976
3977
3978	/** \#TS(err) - 0a. */
3979	VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3980	{
3981	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3982	uSel & X86_SEL_MASK_OFF_RPL, 0);
3983	}
3984
3985
3986	/** \#NP(err) - 0b. */
3987	VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
3988	{
3989	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
3990	}
3991
3992
3993	/** \#NP(sel) - 0b. */
3994	VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
3995	{
3996	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
3997	uSel & ~X86_SEL_RPL, 0);
3998	}
3999
4000
4001	/** \#SS(seg) - 0c. */
4002	VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
4003	{
4004	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4005	uSel & ~X86_SEL_RPL, 0);
4006	}
4007
4008
4009	/** \#SS(err) - 0c. */
4010	VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4011	{
4012	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4013	}
4014
4015
4016	/** \#GP(n) - 0d. */
4017	VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
4018	{
4019	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErr, 0);
4020	}
4021
4022
4023	/** \#GP(0) - 0d. */
4024	VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
4025	{
4026	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4027	}
4028
4029	#ifdef IEM_WITH_SETJMP
4030	/** \#GP(0) - 0d. */
4031	DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4032	{
4033	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4034	}
4035	#endif
4036
4037
4038	/** \#GP(sel) - 0d. */
4039	VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4040	{
4041	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
4042	Sel & ~X86_SEL_RPL, 0);
4043	}
4044
4045
4046	/** \#GP(0) - 0d. */
4047	VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT
4048	{
4049	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4050	}
4051
4052
4053	/** \#GP(sel) - 0d. */
4054	VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4055	{
4056	NOREF(iSegReg); NOREF(fAccess);
4057	return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4058	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4059	}
4060
4061	#ifdef IEM_WITH_SETJMP
4062	/** \#GP(sel) - 0d, longjmp. */
4063	DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4064	{
4065	NOREF(iSegReg); NOREF(fAccess);
4066	iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
4067	IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4068	}
4069	#endif
4070
4071	/** \#GP(sel) - 0d. */
4072	VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4073	{
4074	NOREF(Sel);
4075	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4076	}
4077
4078	#ifdef IEM_WITH_SETJMP
4079	/** \#GP(sel) - 0d, longjmp. */
4080	DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
4081	{
4082	NOREF(Sel);
4083	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4084	}
4085	#endif
4086
4087
4088	/** \#GP(sel) - 0d. */
4089	VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4090	{
4091	NOREF(iSegReg); NOREF(fAccess);
4092	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4093	}
4094
4095	#ifdef IEM_WITH_SETJMP
4096	/** \#GP(sel) - 0d, longjmp. */
4097	DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
4098	{
4099	NOREF(iSegReg); NOREF(fAccess);
4100	iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4101	}
4102	#endif
4103
4104
4105	/** \#PF(n) - 0e. */
4106	VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4107	{
4108	uint16_t uErr;
4109	switch (rc)
4110	{
4111	case VERR_PAGE_NOT_PRESENT:
4112	case VERR_PAGE_TABLE_NOT_PRESENT:
4113	case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
4114	case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
4115	uErr = 0;
4116	break;
4117
4118	default:
4119	AssertMsgFailed(("%Rrc\n", rc));
4120	RT_FALL_THRU();
4121	case VERR_ACCESS_DENIED:
4122	uErr = X86_TRAP_PF_P;
4123	break;
4124
4125	/** @todo reserved */
4126	}
4127
4128	if (pVCpu->iem.s.uCpl == 3)
4129	uErr \|= X86_TRAP_PF_US;
4130
4131	if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
4132	&& ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
4133	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
4134	uErr \|= X86_TRAP_PF_ID;
4135
4136	#if 0 /* This is so much non-sense, really. Why was it done like that? */
4137	/* Note! RW access callers reporting a WRITE protection fault, will clear
4138	the READ flag before calling. So, read-modify-write accesses (RW)
4139	can safely be reported as READ faults. */
4140	if ((fAccess & (IEM_ACCESS_TYPE_WRITE \| IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
4141	uErr \|= X86_TRAP_PF_RW;
4142	#else
4143	if (fAccess & IEM_ACCESS_TYPE_WRITE)
4144	{
4145	/// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
4146	/// (regardless of outcome of the comparison in the latter case).
4147	//if (!(fAccess & IEM_ACCESS_TYPE_READ))
4148	uErr \|= X86_TRAP_PF_RW;
4149	}
4150	#endif
4151
4152	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR \| IEM_XCPT_FLAGS_CR2,
4153	uErr, GCPtrWhere);
4154	}
4155
4156	#ifdef IEM_WITH_SETJMP
4157	/** \#PF(n) - 0e, longjmp. */
4158	DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc) RT_NOEXCEPT
4159	{
4160	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
4161	}
4162	#endif
4163
4164
4165	/** \#MF(0) - 10. */
4166	VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu)
4167	{
4168	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4169	}
4170
4171
4172	/** \#AC(0) - 11. */
4173	VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
4174	{
4175	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, 0, 0);
4176	}
4177
4178	#ifdef IEM_WITH_SETJMP
4179	/** \#AC(0) - 11, longjmp. */
4180	DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) RT_NOEXCEPT
4181	{
4182	longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu)));
4183	}
4184	#endif
4185
4186
4187	/** \#XF(0)/\#XM(0) - 19. */
4188	VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT
4189	{
4190	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4191	}
4192
4193
4194	/** Accessed via IEMOP_RAISE_DIVIDE_ERROR. */
4195	IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
4196	{
4197	NOREF(cbInstr);
4198	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4199	}
4200
4201
4202	/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */
4203	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
4204	{
4205	NOREF(cbInstr);
4206	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4207	}
4208
4209
4210	/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */
4211	IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
4212	{
4213	NOREF(cbInstr);
4214	return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
4215	}
4216
4217
4218	/** @} */
4219
4220	/** @name Common opcode decoders.
4221	* @{
4222	*/
4223	//#include <iprt/mem.h>
4224
4225	/**
4226	* Used to add extra details about a stub case.
4227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4228	*/
4229	void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT
4230	{
4231	#if defined(LOG_ENABLED) && defined(IN_RING3)
4232	PVM pVM = pVCpu->CTX_SUFF(pVM);
4233	char szRegs[4096];
4234	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
4235	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
4236	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
4237	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
4238	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
4239	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
4240	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
4241	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
4242	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
4243	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
4244	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
4245	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
4246	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
4247	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
4248	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
4249	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
4250	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
4251	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
4252	" efer=%016VR{efer}\n"
4253	" pat=%016VR{pat}\n"
4254	" sf_mask=%016VR{sf_mask}\n"
4255	"krnl_gs_base=%016VR{krnl_gs_base}\n"
4256	" lstar=%016VR{lstar}\n"
4257	" star=%016VR{star} cstar=%016VR{cstar}\n"
4258	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
4259	);
4260
4261	char szInstr[256];
4262	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
4263	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
4264	szInstr, sizeof(szInstr), NULL);
4265
4266	RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
4267	#else
4268	RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
4269	#endif
4270	}
4271
4272	/** @} */
4273
4274
4275
4276	/** @name Register Access.
4277	* @{
4278	*/
4279
4280	/**
4281	* Adds a 8-bit signed jump offset to RIP/EIP/IP.
4282	*
4283	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4284	* segment limit.
4285	*
4286	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4287	* @param offNextInstr The offset of the next instruction.
4288	*/
4289	VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr) RT_NOEXCEPT
4290	{
4291	switch (pVCpu->iem.s.enmEffOpSize)
4292	{
4293	case IEMMODE_16BIT:
4294	{
4295	uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4296	if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4297	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4298	return iemRaiseGeneralProtectionFault0(pVCpu);
4299	pVCpu->cpum.GstCtx.rip = uNewIp;
4300	break;
4301	}
4302
4303	case IEMMODE_32BIT:
4304	{
4305	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4306	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4307
4308	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4309	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4310	return iemRaiseGeneralProtectionFault0(pVCpu);
4311	pVCpu->cpum.GstCtx.rip = uNewEip;
4312	break;
4313	}
4314
4315	case IEMMODE_64BIT:
4316	{
4317	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4318
4319	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4320	if (!IEM_IS_CANONICAL(uNewRip))
4321	return iemRaiseGeneralProtectionFault0(pVCpu);
4322	pVCpu->cpum.GstCtx.rip = uNewRip;
4323	break;
4324	}
4325
4326	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4327	}
4328
4329	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4330
4331	#ifndef IEM_WITH_CODE_TLB
4332	/* Flush the prefetch buffer. */
4333	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4334	#endif
4335
4336	return VINF_SUCCESS;
4337	}
4338
4339
4340	/**
4341	* Adds a 16-bit signed jump offset to RIP/EIP/IP.
4342	*
4343	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4344	* segment limit.
4345	*
4346	* @returns Strict VBox status code.
4347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4348	* @param offNextInstr The offset of the next instruction.
4349	*/
4350	VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr) RT_NOEXCEPT
4351	{
4352	Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
4353
4354	uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4355	if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
4356	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4357	return iemRaiseGeneralProtectionFault0(pVCpu);
4358	/** @todo Test 16-bit jump in 64-bit mode. possible? */
4359	pVCpu->cpum.GstCtx.rip = uNewIp;
4360	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4361
4362	#ifndef IEM_WITH_CODE_TLB
4363	/* Flush the prefetch buffer. */
4364	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4365	#endif
4366
4367	return VINF_SUCCESS;
4368	}
4369
4370
4371	/**
4372	* Adds a 32-bit signed jump offset to RIP/EIP/IP.
4373	*
4374	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4375	* segment limit.
4376	*
4377	* @returns Strict VBox status code.
4378	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4379	* @param offNextInstr The offset of the next instruction.
4380	*/
4381	VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr) RT_NOEXCEPT
4382	{
4383	Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
4384
4385	if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
4386	{
4387	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4388
4389	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4390	if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
4391	return iemRaiseGeneralProtectionFault0(pVCpu);
4392	pVCpu->cpum.GstCtx.rip = uNewEip;
4393	}
4394	else
4395	{
4396	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4397
4398	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
4399	if (!IEM_IS_CANONICAL(uNewRip))
4400	return iemRaiseGeneralProtectionFault0(pVCpu);
4401	pVCpu->cpum.GstCtx.rip = uNewRip;
4402	}
4403	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4404
4405	#ifndef IEM_WITH_CODE_TLB
4406	/* Flush the prefetch buffer. */
4407	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4408	#endif
4409
4410	return VINF_SUCCESS;
4411	}
4412
4413
4414	/**
4415	* Performs a near jump to the specified address.
4416	*
4417	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
4418	* segment limit.
4419	*
4420	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4421	* @param uNewRip The new RIP value.
4422	*/
4423	VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
4424	{
4425	switch (pVCpu->iem.s.enmEffOpSize)
4426	{
4427	case IEMMODE_16BIT:
4428	{
4429	Assert(uNewRip <= UINT16_MAX);
4430	if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
4431	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
4432	return iemRaiseGeneralProtectionFault0(pVCpu);
4433	/** @todo Test 16-bit jump in 64-bit mode. */
4434	pVCpu->cpum.GstCtx.rip = uNewRip;
4435	break;
4436	}
4437
4438	case IEMMODE_32BIT:
4439	{
4440	Assert(uNewRip <= UINT32_MAX);
4441	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
4442	Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
4443
4444	if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
4445	return iemRaiseGeneralProtectionFault0(pVCpu);
4446	pVCpu->cpum.GstCtx.rip = uNewRip;
4447	break;
4448	}
4449
4450	case IEMMODE_64BIT:
4451	{
4452	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
4453
4454	if (!IEM_IS_CANONICAL(uNewRip))
4455	return iemRaiseGeneralProtectionFault0(pVCpu);
4456	pVCpu->cpum.GstCtx.rip = uNewRip;
4457	break;
4458	}
4459
4460	IEM_NOT_REACHED_DEFAULT_CASE_RET();
4461	}
4462
4463	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
4464
4465	#ifndef IEM_WITH_CODE_TLB
4466	/* Flush the prefetch buffer. */
4467	pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
4468	#endif
4469
4470	return VINF_SUCCESS;
4471	}
4472
4473	/** @} */
4474
4475
4476	/** @name FPU access and helpers.
4477	*
4478	* @{
4479	*/
4480
4481	/**
4482	* Updates the x87.DS and FPUDP registers.
4483	*
4484	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4485	* @param pFpuCtx The FPU context.
4486	* @param iEffSeg The effective segment register.
4487	* @param GCPtrEff The effective address relative to @a iEffSeg.
4488	*/
4489	DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
4490	{
4491	RTSEL sel;
4492	switch (iEffSeg)
4493	{
4494	case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
4495	case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
4496	case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
4497	case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
4498	case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
4499	case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
4500	default:
4501	AssertMsgFailed(("%d\n", iEffSeg));
4502	sel = pVCpu->cpum.GstCtx.ds.Sel;
4503	}
4504	/** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
4505	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
4506	{
4507	pFpuCtx->DS = 0;
4508	pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
4509	}
4510	else if (!IEM_IS_LONG_MODE(pVCpu))
4511	{
4512	pFpuCtx->DS = sel;
4513	pFpuCtx->FPUDP = GCPtrEff;
4514	}
4515	else
4516	(uint64_t )&pFpuCtx->FPUDP = GCPtrEff;
4517	}
4518
4519
4520	/**
4521	* Rotates the stack registers in the push direction.
4522	*
4523	* @param pFpuCtx The FPU context.
4524	* @remarks This is a complete waste of time, but fxsave stores the registers in
4525	* stack order.
4526	*/
4527	DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
4528	{
4529	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
4530	pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
4531	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
4532	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
4533	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
4534	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
4535	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
4536	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
4537	pFpuCtx->aRegs[0].r80 = r80Tmp;
4538	}
4539
4540
4541	/**
4542	* Rotates the stack registers in the pop direction.
4543	*
4544	* @param pFpuCtx The FPU context.
4545	* @remarks This is a complete waste of time, but fxsave stores the registers in
4546	* stack order.
4547	*/
4548	DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
4549	{
4550	RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
4551	pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
4552	pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
4553	pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
4554	pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
4555	pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
4556	pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
4557	pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
4558	pFpuCtx->aRegs[7].r80 = r80Tmp;
4559	}
4560
4561
4562	/**
4563	* Updates FSW and pushes a FPU result onto the FPU stack if no pending
4564	* exception prevents it.
4565	*
4566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4567	* @param pResult The FPU operation result to push.
4568	* @param pFpuCtx The FPU context.
4569	*/
4570	static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4571	{
4572	/* Update FSW and bail if there are pending exceptions afterwards. */
4573	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4574	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4575	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4576	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4577	{
4578	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES))
4579	Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n",
4580	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4581	pFpuCtx->FSW = fFsw;
4582	return;
4583	}
4584
4585	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4586	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4587	{
4588	/* All is fine, push the actual value. */
4589	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4590	pFpuCtx->aRegs[7].r80 = pResult->r80Result;
4591	}
4592	else if (pFpuCtx->FCW & X86_FCW_IM)
4593	{
4594	/* Masked stack overflow, push QNaN. */
4595	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4596	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4597	}
4598	else
4599	{
4600	/* Raise stack overflow, don't push anything. */
4601	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4602	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4603	Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4604	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4605	return;
4606	}
4607
4608	fFsw &= ~X86_FSW_TOP_MASK;
4609	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4610	pFpuCtx->FSW = fFsw;
4611
4612	iemFpuRotateStackPush(pFpuCtx);
4613	RT_NOREF(pVCpu);
4614	}
4615
4616
4617	/**
4618	* Stores a result in a FPU register and updates the FSW and FTW.
4619	*
4620	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4621	* @param pFpuCtx The FPU context.
4622	* @param pResult The result to store.
4623	* @param iStReg Which FPU register to store it in.
4624	*/
4625	static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4626	{
4627	Assert(iStReg < 8);
4628	uint16_t fNewFsw = pFpuCtx->FSW;
4629	uint16_t const iReg = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK;
4630	fNewFsw &= ~X86_FSW_C_MASK;
4631	fNewFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4632	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4633	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4634	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4635	pFpuCtx->FSW = fNewFsw;
4636	pFpuCtx->FTW \|= RT_BIT(iReg);
4637	pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
4638	RT_NOREF(pVCpu);
4639	}
4640
4641
4642	/**
4643	* Only updates the FPU status word (FSW) with the result of the current
4644	* instruction.
4645	*
4646	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4647	* @param pFpuCtx The FPU context.
4648	* @param u16FSW The FSW output of the current instruction.
4649	*/
4650	static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT
4651	{
4652	uint16_t fNewFsw = pFpuCtx->FSW;
4653	fNewFsw &= ~X86_FSW_C_MASK;
4654	fNewFsw \|= u16FSW & ~X86_FSW_TOP_MASK;
4655	if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4656	Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
4657	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
4658	pFpuCtx->FSW = fNewFsw;
4659	RT_NOREF(pVCpu);
4660	}
4661
4662
4663	/**
4664	* Pops one item off the FPU stack if no pending exception prevents it.
4665	*
4666	* @param pFpuCtx The FPU context.
4667	*/
4668	static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
4669	{
4670	/* Check pending exceptions. */
4671	uint16_t uFSW = pFpuCtx->FSW;
4672	if ( (pFpuCtx->FSW & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4673	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4674	return;
4675
4676	/* TOP--. */
4677	uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
4678	uFSW &= ~X86_FSW_TOP_MASK;
4679	uFSW \|= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
4680	pFpuCtx->FSW = uFSW;
4681
4682	/* Mark the previous ST0 as empty. */
4683	iOldTop >>= X86_FSW_TOP_SHIFT;
4684	pFpuCtx->FTW &= ~RT_BIT(iOldTop);
4685
4686	/* Rotate the registers. */
4687	iemFpuRotateStackPop(pFpuCtx);
4688	}
4689
4690
4691	/**
4692	* Pushes a FPU result onto the FPU stack if no pending exception prevents it.
4693	*
4694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4695	* @param pResult The FPU operation result to push.
4696	*/
4697	void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT
4698	{
4699	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4700	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4701	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4702	}
4703
4704
4705	/**
4706	* Pushes a FPU result onto the FPU stack if no pending exception prevents it,
4707	* and sets FPUDP and FPUDS.
4708	*
4709	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4710	* @param pResult The FPU operation result to push.
4711	* @param iEffSeg The effective segment register.
4712	* @param GCPtrEff The effective address relative to @a iEffSeg.
4713	*/
4714	void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4715	{
4716	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4717	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4718	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4719	iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
4720	}
4721
4722
4723	/**
4724	* Replace ST0 with the first value and push the second onto the FPU stack,
4725	* unless a pending exception prevents it.
4726	*
4727	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4728	* @param pResult The FPU operation result to store and push.
4729	*/
4730	void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT
4731	{
4732	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4733	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4734
4735	/* Update FSW and bail if there are pending exceptions afterwards. */
4736	uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
4737	fFsw \|= pResult->FSW & ~X86_FSW_TOP_MASK;
4738	if ( (fFsw & (X86_FSW_IE \| X86_FSW_ZE \| X86_FSW_DE))
4739	& ~(pFpuCtx->FCW & (X86_FCW_IM \| X86_FCW_ZM \| X86_FCW_DM)))
4740	{
4741	if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
4742	Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n",
4743	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
4744	pFpuCtx->FSW = fFsw;
4745	return;
4746	}
4747
4748	uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
4749	if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
4750	{
4751	/* All is fine, push the actual value. */
4752	pFpuCtx->FTW \|= RT_BIT(iNewTop);
4753	pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
4754	pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
4755	}
4756	else if (pFpuCtx->FCW & X86_FCW_IM)
4757	{
4758	/* Masked stack overflow, push QNaN. */
4759	fFsw \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1;
4760	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
4761	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
4762	}
4763	else
4764	{
4765	/* Raise stack overflow, don't push anything. */
4766	pFpuCtx->FSW \|= pResult->FSW & ~X86_FSW_C_MASK;
4767	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_C1 \| X86_FSW_B \| X86_FSW_ES;
4768	Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n",
4769	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4770	return;
4771	}
4772
4773	fFsw &= ~X86_FSW_TOP_MASK;
4774	fFsw \|= iNewTop << X86_FSW_TOP_SHIFT;
4775	pFpuCtx->FSW = fFsw;
4776
4777	iemFpuRotateStackPush(pFpuCtx);
4778	}
4779
4780
4781	/**
4782	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4783	* FOP.
4784	*
4785	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4786	* @param pResult The result to store.
4787	* @param iStReg Which FPU register to store it in.
4788	*/
4789	void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4790	{
4791	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4792	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4793	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4794	}
4795
4796
4797	/**
4798	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
4799	* FOP, and then pops the stack.
4800	*
4801	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4802	* @param pResult The result to store.
4803	* @param iStReg Which FPU register to store it in.
4804	*/
4805	void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
4806	{
4807	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4808	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4809	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4810	iemFpuMaybePopOne(pFpuCtx);
4811	}
4812
4813
4814	/**
4815	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4816	* FPUDP, and FPUDS.
4817	*
4818	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4819	* @param pResult The result to store.
4820	* @param iStReg Which FPU register to store it in.
4821	* @param iEffSeg The effective memory operand selector register.
4822	* @param GCPtrEff The effective memory operand offset.
4823	*/
4824	void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
4825	uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4826	{
4827	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4828	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4829	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4830	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4831	}
4832
4833
4834	/**
4835	* Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
4836	* FPUDP, and FPUDS, and then pops the stack.
4837	*
4838	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4839	* @param pResult The result to store.
4840	* @param iStReg Which FPU register to store it in.
4841	* @param iEffSeg The effective memory operand selector register.
4842	* @param GCPtrEff The effective memory operand offset.
4843	*/
4844	void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
4845	uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4846	{
4847	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4848	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4849	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4850	iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
4851	iemFpuMaybePopOne(pFpuCtx);
4852	}
4853
4854
4855	/**
4856	* Updates the FOP, FPUIP, and FPUCS. For FNOP.
4857	*
4858	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4859	*/
4860	void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT
4861	{
4862	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4863	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4864	}
4865
4866
4867	/**
4868	* Updates the FSW, FOP, FPUIP, and FPUCS.
4869	*
4870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4871	* @param u16FSW The FSW from the current instruction.
4872	*/
4873	void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4874	{
4875	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4876	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4877	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4878	}
4879
4880
4881	/**
4882	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
4883	*
4884	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4885	* @param u16FSW The FSW from the current instruction.
4886	*/
4887	void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4888	{
4889	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4890	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4891	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4892	iemFpuMaybePopOne(pFpuCtx);
4893	}
4894
4895
4896	/**
4897	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
4898	*
4899	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4900	* @param u16FSW The FSW from the current instruction.
4901	* @param iEffSeg The effective memory operand selector register.
4902	* @param GCPtrEff The effective memory operand offset.
4903	*/
4904	void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4905	{
4906	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4907	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4908	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4909	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4910	}
4911
4912
4913	/**
4914	* Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
4915	*
4916	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4917	* @param u16FSW The FSW from the current instruction.
4918	*/
4919	void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
4920	{
4921	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4922	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4923	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4924	iemFpuMaybePopOne(pFpuCtx);
4925	iemFpuMaybePopOne(pFpuCtx);
4926	}
4927
4928
4929	/**
4930	* Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
4931	*
4932	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4933	* @param u16FSW The FSW from the current instruction.
4934	* @param iEffSeg The effective memory operand selector register.
4935	* @param GCPtrEff The effective memory operand offset.
4936	*/
4937	void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4938	{
4939	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4940	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
4941	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4942	iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
4943	iemFpuMaybePopOne(pFpuCtx);
4944	}
4945
4946
4947	/**
4948	* Worker routine for raising an FPU stack underflow exception.
4949	*
4950	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4951	* @param pFpuCtx The FPU context.
4952	* @param iStReg The stack register being accessed.
4953	*/
4954	static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg)
4955	{
4956	Assert(iStReg < 8 \|\| iStReg == UINT8_MAX);
4957	if (pFpuCtx->FCW & X86_FCW_IM)
4958	{
4959	/* Masked underflow. */
4960	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4961	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
4962	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
4963	if (iStReg != UINT8_MAX)
4964	{
4965	pFpuCtx->FTW \|= RT_BIT(iReg);
4966	iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
4967	}
4968	}
4969	else
4970	{
4971	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
4972	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
4973	Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n",
4974	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
4975	}
4976	RT_NOREF(pVCpu);
4977	}
4978
4979
4980	/**
4981	* Raises a FPU stack underflow exception.
4982	*
4983	* @param pVCpu The cross context virtual CPU structure of the calling thread.
4984	* @param iStReg The destination register that should be loaded
4985	* with QNaN if \#IS is not masked. Specify
4986	* UINT8_MAX if none (like for fcom).
4987	*/
4988	void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
4989	{
4990	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4991	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
4992	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
4993	}
4994
4995
4996	void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
4997	{
4998	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
4999	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5000	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5001	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5002	}
5003
5004
5005	void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
5006	{
5007	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5008	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5009	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5010	iemFpuMaybePopOne(pFpuCtx);
5011	}
5012
5013
5014	void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5015	{
5016	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5017	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5018	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5019	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
5020	iemFpuMaybePopOne(pFpuCtx);
5021	}
5022
5023
5024	void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT
5025	{
5026	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5027	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5028	iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX);
5029	iemFpuMaybePopOne(pFpuCtx);
5030	iemFpuMaybePopOne(pFpuCtx);
5031	}
5032
5033
5034	void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5035	{
5036	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5037	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5038
5039	if (pFpuCtx->FCW & X86_FCW_IM)
5040	{
5041	/* Masked overflow - Push QNaN. */
5042	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5043	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5044	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5045	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5046	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5047	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5048	iemFpuRotateStackPush(pFpuCtx);
5049	}
5050	else
5051	{
5052	/* Exception pending - don't change TOP or the register stack. */
5053	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5054	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5055	Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n",
5056	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5057	}
5058	}
5059
5060
5061	void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT
5062	{
5063	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5064	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5065
5066	if (pFpuCtx->FCW & X86_FCW_IM)
5067	{
5068	/* Masked overflow - Push QNaN. */
5069	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5070	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5071	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF;
5072	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5073	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5074	iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
5075	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5076	iemFpuRotateStackPush(pFpuCtx);
5077	}
5078	else
5079	{
5080	/* Exception pending - don't change TOP or the register stack. */
5081	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5082	pFpuCtx->FSW \|= X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5083	Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n",
5084	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5085	}
5086	}
5087
5088
5089	/**
5090	* Worker routine for raising an FPU stack overflow exception on a push.
5091	*
5092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5093	* @param pFpuCtx The FPU context.
5094	*/
5095	static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
5096	{
5097	if (pFpuCtx->FCW & X86_FCW_IM)
5098	{
5099	/* Masked overflow. */
5100	uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
5101	pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK \| X86_FSW_C_MASK);
5102	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF;
5103	pFpuCtx->FSW \|= iNewTop << X86_FSW_TOP_SHIFT;
5104	pFpuCtx->FTW \|= RT_BIT(iNewTop);
5105	iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
5106	iemFpuRotateStackPush(pFpuCtx);
5107	}
5108	else
5109	{
5110	/* Exception pending - don't change TOP or the register stack. */
5111	pFpuCtx->FSW &= ~X86_FSW_C_MASK;
5112	pFpuCtx->FSW \|= X86_FSW_C1 \| X86_FSW_IE \| X86_FSW_SF \| X86_FSW_ES \| X86_FSW_B;
5113	Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n",
5114	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
5115	}
5116	RT_NOREF(pVCpu);
5117	}
5118
5119
5120	/**
5121	* Raises a FPU stack overflow exception on a push.
5122	*
5123	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5124	*/
5125	void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT
5126	{
5127	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5128	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5129	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5130	}
5131
5132
5133	/**
5134	* Raises a FPU stack overflow exception on a push with a memory operand.
5135	*
5136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5137	* @param iEffSeg The effective memory operand selector register.
5138	* @param GCPtrEff The effective memory operand offset.
5139	*/
5140	void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
5141	{
5142	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5143	iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
5144	iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
5145	iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
5146	}
5147
5148	/** @} */
5149
5150
5151	/** @name SSE+AVX SIMD access and helpers.
5152	*
5153	* @{
5154	*/
5155	/**
5156	* Stores a result in a SIMD XMM register, updates the MXCSR.
5157	*
5158	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5159	* @param pResult The result to store.
5160	* @param iXmmReg Which SIMD XMM register to store the result in.
5161	*/
5162	void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT
5163	{
5164	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5165	pFpuCtx->MXCSR \|= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS;
5166
5167	/* The result is only updated if there is no unmasked exception pending. */
5168	if (( ~((pFpuCtx->MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
5169	& (pFpuCtx->MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0)
5170	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult;
5171	}
5172
5173
5174	/**
5175	* Updates the MXCSR.
5176	*
5177	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5178	* @param fMxcsr The new MXCSR value.
5179	*/
5180	void iemSseUpdateMxcsr(PVMCPUCC pVCpu, uint32_t fMxcsr) RT_NOEXCEPT
5181	{
5182	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
5183	pFpuCtx->MXCSR \|= fMxcsr & X86_MXCSR_XCPT_FLAGS;
5184	}
5185	/** @} */
5186
5187
5188	/** @name Memory access.
5189	*
5190	* @{
5191	*/
5192
5193
5194	/**
5195	* Updates the IEMCPU::cbWritten counter if applicable.
5196	*
5197	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5198	* @param fAccess The access being accounted for.
5199	* @param cbMem The access size.
5200	*/
5201	DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
5202	{
5203	if ( (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK \| IEM_ACCESS_TYPE_WRITE)
5204	\|\| (fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA \| IEM_ACCESS_TYPE_WRITE) )
5205	pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
5206	}
5207
5208
5209	/**
5210	* Applies the segment limit, base and attributes.
5211	*
5212	* This may raise a \#GP or \#SS.
5213	*
5214	* @returns VBox strict status code.
5215	*
5216	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5217	* @param fAccess The kind of access which is being performed.
5218	* @param iSegReg The index of the segment register to apply.
5219	* This is UINT8_MAX if none (for IDT, GDT, LDT,
5220	* TSS, ++).
5221	* @param cbMem The access size.
5222	* @param pGCPtrMem Pointer to the guest memory address to apply
5223	* segmentation to. Input and output parameter.
5224	*/
5225	VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT
5226	{
5227	if (iSegReg == UINT8_MAX)
5228	return VINF_SUCCESS;
5229
5230	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
5231	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
5232	switch (pVCpu->iem.s.enmCpuMode)
5233	{
5234	case IEMMODE_16BIT:
5235	case IEMMODE_32BIT:
5236	{
5237	RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
5238	RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
5239
5240	if ( pSel->Attr.n.u1Present
5241	&& !pSel->Attr.n.u1Unusable)
5242	{
5243	Assert(pSel->Attr.n.u1DescType);
5244	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
5245	{
5246	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5247	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
5248	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5249
5250	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5251	{
5252	/** @todo CPL check. */
5253	}
5254
5255	/*
5256	* There are two kinds of data selectors, normal and expand down.
5257	*/
5258	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
5259	{
5260	if ( GCPtrFirst32 > pSel->u32Limit
5261	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5262	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5263	}
5264	else
5265	{
5266	/*
5267	* The upper boundary is defined by the B bit, not the G bit!
5268	*/
5269	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
5270	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
5271	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5272	}
5273	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5274	}
5275	else
5276	{
5277	/*
5278	* Code selector and usually be used to read thru, writing is
5279	* only permitted in real and V8086 mode.
5280	*/
5281	if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5282	\|\| ( (fAccess & IEM_ACCESS_TYPE_READ)
5283	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
5284	&& !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
5285	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
5286
5287	if ( GCPtrFirst32 > pSel->u32Limit
5288	\|\| GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
5289	return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
5290
5291	if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
5292	{
5293	/** @todo CPL check. */
5294	}
5295
5296	*pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
5297	}
5298	}
5299	else
5300	return iemRaiseGeneralProtectionFault0(pVCpu);
5301	return VINF_SUCCESS;
5302	}
5303
5304	case IEMMODE_64BIT:
5305	{
5306	RTGCPTR GCPtrMem = *pGCPtrMem;
5307	if (iSegReg == X86_SREG_GS \|\| iSegReg == X86_SREG_FS)
5308	*pGCPtrMem = GCPtrMem + pSel->u64Base;
5309
5310	Assert(cbMem >= 1);
5311	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
5312	return VINF_SUCCESS;
5313	/** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
5314	* 4.12.2 "Data Limit Checks in 64-bit Mode". */
5315	return iemRaiseGeneralProtectionFault0(pVCpu);
5316	}
5317
5318	default:
5319	AssertFailedReturn(VERR_IEM_IPE_7);
5320	}
5321	}
5322
5323
5324	/**
5325	* Translates a virtual address to a physical physical address and checks if we
5326	* can access the page as specified.
5327	*
5328	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5329	* @param GCPtrMem The virtual address.
5330	* @param fAccess The intended access.
5331	* @param pGCPhysMem Where to return the physical address.
5332	*/
5333	VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT
5334	{
5335	/** @todo Need a different PGM interface here. We're currently using
5336	* generic / REM interfaces. this won't cut it for R0. */
5337	/** @todo If/when PGM handles paged real-mode, we can remove the hack in
5338	* iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
5339	* here. */
5340	PGMPTWALK Walk;
5341	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
5342	if (RT_FAILURE(rc))
5343	{
5344	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
5345	/** @todo Check unassigned memory in unpaged mode. */
5346	/** @todo Reserved bits in page tables. Requires new PGM interface. */
5347	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5348	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5349	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
5350	#endif
5351	*pGCPhysMem = NIL_RTGCPHYS;
5352	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
5353	}
5354
5355	/* If the page is writable and does not have the no-exec bit set, all
5356	access is allowed. Otherwise we'll have to check more carefully... */
5357	if ((Walk.fEffective & (X86_PTE_RW \| X86_PTE_US \| X86_PTE_PAE_NX)) != (X86_PTE_RW \| X86_PTE_US))
5358	{
5359	/* Write to read only memory? */
5360	if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
5361	&& !(Walk.fEffective & X86_PTE_RW)
5362	&& ( ( pVCpu->iem.s.uCpl == 3
5363	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5364	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
5365	{
5366	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
5367	*pGCPhysMem = NIL_RTGCPHYS;
5368	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5369	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5370	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5371	#endif
5372	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
5373	}
5374
5375	/* Kernel memory accessed by userland? */
5376	if ( !(Walk.fEffective & X86_PTE_US)
5377	&& pVCpu->iem.s.uCpl == 3
5378	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
5379	{
5380	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
5381	*pGCPhysMem = NIL_RTGCPHYS;
5382	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5383	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5384	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5385	#endif
5386	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
5387	}
5388
5389	/* Executing non-executable memory? */
5390	if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
5391	&& (Walk.fEffective & X86_PTE_PAE_NX)
5392	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
5393	{
5394	Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
5395	*pGCPhysMem = NIL_RTGCPHYS;
5396	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
5397	if (Walk.fFailed & PGM_WALKFAIL_EPT)
5398	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
5399	#endif
5400	return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE),
5401	VERR_ACCESS_DENIED);
5402	}
5403	}
5404
5405	/*
5406	* Set the dirty / access flags.
5407	* ASSUMES this is set when the address is translated rather than on committ...
5408	*/
5409	/** @todo testcase: check when A and D bits are actually set by the CPU. */
5410	uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
5411	if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
5412	{
5413	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
5414	AssertRC(rc2);
5415	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
5416	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
5417	}
5418
5419	RTGCPHYS const GCPhys = Walk.GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
5420	*pGCPhysMem = GCPhys;
5421	return VINF_SUCCESS;
5422	}
5423
5424
5425	/**
5426	* Looks up a memory mapping entry.
5427	*
5428	* @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
5429	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5430	* @param pvMem The memory address.
5431	* @param fAccess The access to.
5432	*/
5433	DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
5434	{
5435	Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5436	fAccess &= IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK;
5437	if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
5438	&& (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5439	return 0;
5440	if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
5441	&& (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5442	return 1;
5443	if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
5444	&& (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK \| IEM_ACCESS_TYPE_MASK)) == fAccess)
5445	return 2;
5446	return VERR_NOT_FOUND;
5447	}
5448
5449
5450	/**
5451	* Finds a free memmap entry when using iNextMapping doesn't work.
5452	*
5453	* @returns Memory mapping index, 1024 on failure.
5454	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5455	*/
5456	static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
5457	{
5458	/*
5459	* The easy case.
5460	*/
5461	if (pVCpu->iem.s.cActiveMappings == 0)
5462	{
5463	pVCpu->iem.s.iNextMapping = 1;
5464	return 0;
5465	}
5466
5467	/* There should be enough mappings for all instructions. */
5468	AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
5469
5470	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
5471	if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
5472	return i;
5473
5474	AssertFailedReturn(1024);
5475	}
5476
5477
5478	/**
5479	* Commits a bounce buffer that needs writing back and unmaps it.
5480	*
5481	* @returns Strict VBox status code.
5482	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5483	* @param iMemMap The index of the buffer to commit.
5484	* @param fPostponeFail Whether we can postpone writer failures to ring-3.
5485	* Always false in ring-3, obviously.
5486	*/
5487	static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
5488	{
5489	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
5490	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
5491	#ifdef IN_RING3
5492	Assert(!fPostponeFail);
5493	RT_NOREF_PV(fPostponeFail);
5494	#endif
5495
5496	/*
5497	* Do the writing.
5498	*/
5499	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5500	if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
5501	{
5502	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
5503	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5504	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5505	if (!pVCpu->iem.s.fBypassHandlers)
5506	{
5507	/*
5508	* Carefully and efficiently dealing with access handler return
5509	* codes make this a little bloated.
5510	*/
5511	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
5512	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5513	pbBuf,
5514	cbFirst,
5515	PGMACCESSORIGIN_IEM);
5516	if (rcStrict == VINF_SUCCESS)
5517	{
5518	if (cbSecond)
5519	{
5520	rcStrict = PGMPhysWrite(pVM,
5521	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5522	pbBuf + cbFirst,
5523	cbSecond,
5524	PGMACCESSORIGIN_IEM);
5525	if (rcStrict == VINF_SUCCESS)
5526	{ /* nothing */ }
5527	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5528	{
5529	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
5530	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5531	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5532	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5533	}
5534	#ifndef IN_RING3
5535	else if (fPostponeFail)
5536	{
5537	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5538	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5539	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5540	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5541	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5542	return iemSetPassUpStatus(pVCpu, rcStrict);
5543	}
5544	#endif
5545	else
5546	{
5547	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5548	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5549	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5550	return rcStrict;
5551	}
5552	}
5553	}
5554	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5555	{
5556	if (!cbSecond)
5557	{
5558	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
5559	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5560	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5561	}
5562	else
5563	{
5564	VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
5565	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5566	pbBuf + cbFirst,
5567	cbSecond,
5568	PGMACCESSORIGIN_IEM);
5569	if (rcStrict2 == VINF_SUCCESS)
5570	{
5571	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
5572	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5573	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5574	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5575	}
5576	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5577	{
5578	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
5579	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5580	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5581	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5582	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5583	}
5584	#ifndef IN_RING3
5585	else if (fPostponeFail)
5586	{
5587	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5588	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5589	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5590	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_2ND;
5591	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5592	return iemSetPassUpStatus(pVCpu, rcStrict);
5593	}
5594	#endif
5595	else
5596	{
5597	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5598	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5599	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
5600	return rcStrict2;
5601	}
5602	}
5603	}
5604	#ifndef IN_RING3
5605	else if (fPostponeFail)
5606	{
5607	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
5608	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5609	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5610	if (!cbSecond)
5611	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST;
5612	else
5613	pVCpu->iem.s.aMemMappings[iMemMap].fAccess \|= IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND;
5614	VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
5615	return iemSetPassUpStatus(pVCpu, rcStrict);
5616	}
5617	#endif
5618	else
5619	{
5620	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5621	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
5622	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5623	return rcStrict;
5624	}
5625	}
5626	else
5627	{
5628	/*
5629	* No access handlers, much simpler.
5630	*/
5631	int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
5632	if (RT_SUCCESS(rc))
5633	{
5634	if (cbSecond)
5635	{
5636	rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
5637	if (RT_SUCCESS(rc))
5638	{ /* likely */ }
5639	else
5640	{
5641	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
5642	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
5643	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
5644	return rc;
5645	}
5646	}
5647	}
5648	else
5649	{
5650	Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
5651	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
5652	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
5653	return rc;
5654	}
5655	}
5656	}
5657
5658	#if defined(IEM_LOG_MEMORY_WRITES)
5659	Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
5660	RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
5661	if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
5662	Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
5663	RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
5664	&pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
5665
5666	size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
5667	g_cbIemWrote = cbWrote;
5668	memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
5669	#endif
5670
5671	/*
5672	* Free the mapping entry.
5673	*/
5674	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
5675	Assert(pVCpu->iem.s.cActiveMappings != 0);
5676	pVCpu->iem.s.cActiveMappings--;
5677	return VINF_SUCCESS;
5678	}
5679
5680
5681	/**
5682	* iemMemMap worker that deals with a request crossing pages.
5683	*/
5684	static VBOXSTRICTRC
5685	iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
5686	{
5687	/*
5688	* Do the address translations.
5689	*/
5690	RTGCPHYS GCPhysFirst;
5691	VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
5692	if (rcStrict != VINF_SUCCESS)
5693	return rcStrict;
5694
5695	RTGCPHYS GCPhysSecond;
5696	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
5697	fAccess, &GCPhysSecond);
5698	if (rcStrict != VINF_SUCCESS)
5699	return rcStrict;
5700	GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
5701
5702	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5703
5704	/*
5705	* Read in the current memory content if it's a read, execute or partial
5706	* write access.
5707	*/
5708	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5709	uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (GCPhysFirst & GUEST_PAGE_OFFSET_MASK);
5710	uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
5711
5712	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
5713	{
5714	if (!pVCpu->iem.s.fBypassHandlers)
5715	{
5716	/*
5717	* Must carefully deal with access handler status codes here,
5718	* makes the code a bit bloated.
5719	*/
5720	rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
5721	if (rcStrict == VINF_SUCCESS)
5722	{
5723	rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5724	if (rcStrict == VINF_SUCCESS)
5725	{ /likely / }
5726	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5727	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5728	else
5729	{
5730	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
5731	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
5732	return rcStrict;
5733	}
5734	}
5735	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5736	{
5737	VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
5738	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
5739	{
5740	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
5741	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5742	}
5743	else
5744	{
5745	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
5746	GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
5747	return rcStrict2;
5748	}
5749	}
5750	else
5751	{
5752	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5753	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5754	return rcStrict;
5755	}
5756	}
5757	else
5758	{
5759	/*
5760	* No informational status codes here, much more straight forward.
5761	*/
5762	int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
5763	if (RT_SUCCESS(rc))
5764	{
5765	Assert(rc == VINF_SUCCESS);
5766	rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
5767	if (RT_SUCCESS(rc))
5768	Assert(rc == VINF_SUCCESS);
5769	else
5770	{
5771	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
5772	return rc;
5773	}
5774	}
5775	else
5776	{
5777	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
5778	return rc;
5779	}
5780	}
5781	}
5782	#ifdef VBOX_STRICT
5783	else
5784	memset(pbBuf, 0xcc, cbMem);
5785	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5786	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5787	#endif
5788	AssertCompileMemberAlignment(VMCPU, iem.s.aBounceBuffers, 64);
5789
5790	/*
5791	* Commit the bounce buffer entry.
5792	*/
5793	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5794	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
5795	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
5796	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
5797	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
5798	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5799	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
5800	pVCpu->iem.s.iNextMapping = iMemMap + 1;
5801	pVCpu->iem.s.cActiveMappings++;
5802
5803	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5804	*ppvMem = pbBuf;
5805	return VINF_SUCCESS;
5806	}
5807
5808
5809	/**
5810	* iemMemMap woker that deals with iemMemPageMap failures.
5811	*/
5812	static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
5813	RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
5814	{
5815	/*
5816	* Filter out conditions we can handle and the ones which shouldn't happen.
5817	*/
5818	if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
5819	&& rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
5820	&& rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
5821	{
5822	AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
5823	return rcMap;
5824	}
5825	pVCpu->iem.s.cPotentialExits++;
5826
5827	/*
5828	* Read in the current memory content if it's a read, execute or partial
5829	* write access.
5830	*/
5831	uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
5832	if (fAccess & (IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_EXEC \| IEM_ACCESS_PARTIAL_WRITE))
5833	{
5834	if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
5835	memset(pbBuf, 0xff, cbMem);
5836	else
5837	{
5838	int rc;
5839	if (!pVCpu->iem.s.fBypassHandlers)
5840	{
5841	VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
5842	if (rcStrict == VINF_SUCCESS)
5843	{ /* nothing */ }
5844	else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
5845	rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
5846	else
5847	{
5848	Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5849	GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
5850	return rcStrict;
5851	}
5852	}
5853	else
5854	{
5855	rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
5856	if (RT_SUCCESS(rc))
5857	{ /* likely */ }
5858	else
5859	{
5860	Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
5861	GCPhysFirst, rc));
5862	return rc;
5863	}
5864	}
5865	}
5866	}
5867	#ifdef VBOX_STRICT
5868	else
5869	memset(pbBuf, 0xcc, cbMem);
5870	#endif
5871	#ifdef VBOX_STRICT
5872	if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
5873	memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
5874	#endif
5875
5876	/*
5877	* Commit the bounce buffer entry.
5878	*/
5879	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
5880	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
5881	pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
5882	pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
5883	pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
5884	pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
5885	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess \| IEM_ACCESS_BOUNCE_BUFFERED;
5886	pVCpu->iem.s.iNextMapping = iMemMap + 1;
5887	pVCpu->iem.s.cActiveMappings++;
5888
5889	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
5890	*ppvMem = pbBuf;
5891	return VINF_SUCCESS;
5892	}
5893
5894
5895
5896	/**
5897	* Maps the specified guest memory for the given kind of access.
5898	*
5899	* This may be using bounce buffering of the memory if it's crossing a page
5900	* boundary or if there is an access handler installed for any of it. Because
5901	* of lock prefix guarantees, we're in for some extra clutter when this
5902	* happens.
5903	*
5904	* This may raise a \#GP, \#SS, \#PF or \#AC.
5905	*
5906	* @returns VBox strict status code.
5907	*
5908	* @param pVCpu The cross context virtual CPU structure of the calling thread.
5909	* @param ppvMem Where to return the pointer to the mapped memory.
5910	* @param cbMem The number of bytes to map. This is usually 1, 2, 4, 6,
5911	* 8, 12, 16, 32 or 512. When used by string operations
5912	* it can be up to a page.
5913	* @param iSegReg The index of the segment register to use for this
5914	* access. The base and limits are checked. Use UINT8_MAX
5915	* to indicate that no segmentation is required (for IDT,
5916	* GDT and LDT accesses).
5917	* @param GCPtrMem The address of the guest memory.
5918	* @param fAccess How the memory is being accessed. The
5919	* IEM_ACCESS_TYPE_XXX bit is used to figure out how to map
5920	* the memory, while the IEM_ACCESS_WHAT_XXX bit is used
5921	* when raising exceptions.
5922	* @param uAlignCtl Alignment control:
5923	* - Bits 15:0 is the alignment mask.
5924	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
5925	* IEM_MEMMAP_F_ALIGN_SSE, and
5926	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
5927	* Pass zero to skip alignment.
5928	*/
5929	VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
5930	uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT
5931	{
5932	/*
5933	* Check the input and figure out which mapping entry to use.
5934	*/
5935	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 256 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 102 \|\| cbMem == 94); /* 512 is the max! */
5936	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
5937	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
5938
5939	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
5940	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
5941	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
5942	{
5943	iMemMap = iemMemMapFindFree(pVCpu);
5944	AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
5945	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
5946	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
5947	pVCpu->iem.s.aMemMappings[2].fAccess),
5948	VERR_IEM_IPE_9);
5949	}
5950
5951	/*
5952	* Map the memory, checking that we can actually access it. If something
5953	* slightly complicated happens, fall back on bounce buffering.
5954	*/
5955	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
5956	if (rcStrict == VINF_SUCCESS)
5957	{ /* likely */ }
5958	else
5959	return rcStrict;
5960
5961	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */
5962	{ /* likely */ }
5963	else
5964	return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
5965
5966	/*
5967	* Alignment check.
5968	*/
5969	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
5970	{ /* likelyish */ }
5971	else
5972	{
5973	/* Misaligned access. */
5974	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
5975	{
5976	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
5977	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
5978	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
5979	{
5980	AssertCompile(X86_CR0_AM == X86_EFL_AC);
5981
5982	if (iemMemAreAlignmentChecksEnabled(pVCpu))
5983	return iemRaiseAlignmentCheckException(pVCpu);
5984	}
5985	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
5986	&& iemMemAreAlignmentChecksEnabled(pVCpu)
5987	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
5988	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
5989	)
5990	return iemRaiseAlignmentCheckException(pVCpu);
5991	else
5992	return iemRaiseGeneralProtectionFault0(pVCpu);
5993	}
5994	}
5995
5996	#ifdef IEM_WITH_DATA_TLB
5997	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
5998
5999	/*
6000	* Get the TLB entry for this page.
6001	*/
6002	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6003	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6004	if (pTlbe->uTag == uTag)
6005	{
6006	# ifdef VBOX_WITH_STATISTICS
6007	pVCpu->iem.s.DataTlb.cTlbHits++;
6008	# endif
6009	}
6010	else
6011	{
6012	pVCpu->iem.s.DataTlb.cTlbMisses++;
6013	PGMPTWALK Walk;
6014	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6015	if (RT_FAILURE(rc))
6016	{
6017	Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6018	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6019	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6020	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6021	# endif
6022	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6023	}
6024
6025	Assert(Walk.fSucceeded);
6026	pTlbe->uTag = uTag;
6027	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6028	pTlbe->GCPhys = Walk.GCPhys;
6029	pTlbe->pbMappingR3 = NULL;
6030	}
6031
6032	/*
6033	* Check TLB page table level access flags.
6034	*/
6035	/* If the page is either supervisor only or non-writable, we need to do
6036	more careful access checks. */
6037	if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER \| IEMTLBE_F_PT_NO_WRITE))
6038	{
6039	/* Write to read only memory? */
6040	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
6041	&& (fAccess & IEM_ACCESS_TYPE_WRITE)
6042	&& ( ( pVCpu->iem.s.uCpl == 3
6043	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6044	\|\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
6045	{
6046	Log(("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6047	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6048	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6049	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6050	# endif
6051	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6052	}
6053
6054	/* Kernel memory accessed by userland? */
6055	if ( (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
6056	&& pVCpu->iem.s.uCpl == 3
6057	&& !(fAccess & IEM_ACCESS_WHAT_SYS))
6058	{
6059	Log(("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6060	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6061	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6062	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6063	# endif
6064	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6065	}
6066	}
6067
6068	/*
6069	* Set the dirty / access flags.
6070	* ASSUMES this is set when the address is translated rather than on commit...
6071	*/
6072	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6073	uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) \| IEMTLBE_F_PT_NO_ACCESSED;
6074	if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty)
6075	{
6076	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6077	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6078	AssertRC(rc2);
6079	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6080	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6081	pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty;
6082	}
6083
6084	/*
6085	* Look up the physical page info if necessary.
6086	*/
6087	uint8_t *pbMem = NULL;
6088	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6089	# ifdef IN_RING3
6090	pbMem = pTlbe->pbMappingR3;
6091	# else
6092	pbMem = NULL;
6093	# endif
6094	else
6095	{
6096	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6097	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6098	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6099	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6100	if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
6101	{ /* likely */ }
6102	else
6103	IEMTlbInvalidateAllPhysicalSlow(pVCpu);
6104	pTlbe->pbMappingR3 = NULL;
6105	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6106	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
6107	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6108	&pbMem, &pTlbe->fFlagsAndPhysRev);
6109	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6110	# ifdef IN_RING3
6111	pTlbe->pbMappingR3 = pbMem;
6112	# endif
6113	}
6114
6115	/*
6116	* Check the physical page level access and mapping.
6117	*/
6118	if ( !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))
6119	\|\| !(pTlbe->fFlagsAndPhysRev & ( (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0)
6120	\| (fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0))) )
6121	{ /* probably likely */ }
6122	else
6123	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem,
6124	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6125	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6126	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6127	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6128	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6129
6130	if (pbMem)
6131	{
6132	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6133	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6134	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6135	}
6136	else
6137	{
6138	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6139	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6140	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6141	if (rcStrict != VINF_SUCCESS)
6142	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6143	}
6144
6145	void * const pvMem = pbMem;
6146
6147	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6148	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6149	if (fAccess & IEM_ACCESS_TYPE_READ)
6150	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6151
6152	#else /* !IEM_WITH_DATA_TLB */
6153
6154	RTGCPHYS GCPhysFirst;
6155	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6156	if (rcStrict != VINF_SUCCESS)
6157	return rcStrict;
6158
6159	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6160	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6161	if (fAccess & IEM_ACCESS_TYPE_READ)
6162	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6163
6164	void *pvMem;
6165	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6166	if (rcStrict != VINF_SUCCESS)
6167	return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6168
6169	#endif /* !IEM_WITH_DATA_TLB */
6170
6171	/*
6172	* Fill in the mapping table entry.
6173	*/
6174	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6175	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6176	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6177	pVCpu->iem.s.cActiveMappings += 1;
6178
6179	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6180	*ppvMem = pvMem;
6181
6182	return VINF_SUCCESS;
6183	}
6184
6185
6186	/**
6187	* Commits the guest memory if bounce buffered and unmaps it.
6188	*
6189	* @returns Strict VBox status code.
6190	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6191	* @param pvMem The mapping.
6192	* @param fAccess The kind of access.
6193	*/
6194	VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6195	{
6196	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6197	AssertReturn(iMemMap >= 0, iMemMap);
6198
6199	/* If it's bounce buffered, we may need to write back the buffer. */
6200	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6201	{
6202	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6203	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6204	}
6205	/* Otherwise unlock it. */
6206	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6207	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6208
6209	/* Free the entry. */
6210	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6211	Assert(pVCpu->iem.s.cActiveMappings != 0);
6212	pVCpu->iem.s.cActiveMappings--;
6213	return VINF_SUCCESS;
6214	}
6215
6216	#ifdef IEM_WITH_SETJMP
6217
6218	/**
6219	* Maps the specified guest memory for the given kind of access, longjmp on
6220	* error.
6221	*
6222	* This may be using bounce buffering of the memory if it's crossing a page
6223	* boundary or if there is an access handler installed for any of it. Because
6224	* of lock prefix guarantees, we're in for some extra clutter when this
6225	* happens.
6226	*
6227	* This may raise a \#GP, \#SS, \#PF or \#AC.
6228	*
6229	* @returns Pointer to the mapped memory.
6230	*
6231	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6232	* @param cbMem The number of bytes to map. This is usually 1,
6233	* 2, 4, 6, 8, 12, 16, 32 or 512. When used by
6234	* string operations it can be up to a page.
6235	* @param iSegReg The index of the segment register to use for
6236	* this access. The base and limits are checked.
6237	* Use UINT8_MAX to indicate that no segmentation
6238	* is required (for IDT, GDT and LDT accesses).
6239	* @param GCPtrMem The address of the guest memory.
6240	* @param fAccess How the memory is being accessed. The
6241	* IEM_ACCESS_TYPE_XXX bit is used to figure out
6242	* how to map the memory, while the
6243	* IEM_ACCESS_WHAT_XXX bit is used when raising
6244	* exceptions.
6245	* @param uAlignCtl Alignment control:
6246	* - Bits 15:0 is the alignment mask.
6247	* - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
6248	* IEM_MEMMAP_F_ALIGN_SSE, and
6249	* IEM_MEMMAP_F_ALIGN_GP_OR_AC.
6250	* Pass zero to skip alignment.
6251	*/
6252	void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess,
6253	uint32_t uAlignCtl) RT_NOEXCEPT
6254	{
6255	/*
6256	* Check the input, check segment access and adjust address
6257	* with segment base.
6258	*/
6259	Assert(cbMem <= 64 \|\| cbMem == 512 \|\| cbMem == 108 \|\| cbMem == 104 \|\| cbMem == 94); /* 512 is the max! */
6260	Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK \| IEM_ACCESS_WHAT_MASK)));
6261	Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
6262
6263	VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
6264	if (rcStrict == VINF_SUCCESS) { /likely/ }
6265	else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6266
6267	/*
6268	* Alignment check.
6269	*/
6270	if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
6271	{ /* likelyish */ }
6272	else
6273	{
6274	/* Misaligned access. */
6275	if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6276	{
6277	if ( !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
6278	\|\| ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
6279	&& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
6280	{
6281	AssertCompile(X86_CR0_AM == X86_EFL_AC);
6282
6283	if (iemMemAreAlignmentChecksEnabled(pVCpu))
6284	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6285	}
6286	else if ( (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
6287	&& iemMemAreAlignmentChecksEnabled(pVCpu)
6288	/** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
6289	* implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++. */
6290	)
6291	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6292	else
6293	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
6294	}
6295	}
6296
6297	/*
6298	* Figure out which mapping entry to use.
6299	*/
6300	unsigned iMemMap = pVCpu->iem.s.iNextMapping;
6301	if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
6302	\|\| pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
6303	{
6304	iMemMap = iemMemMapFindFree(pVCpu);
6305	AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
6306	("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
6307	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
6308	pVCpu->iem.s.aMemMappings[2].fAccess),
6309	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
6310	}
6311
6312	/*
6313	* Crossing a page boundary?
6314	*/
6315	if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
6316	{ /* No (likely). */ }
6317	else
6318	{
6319	void *pvMem;
6320	rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
6321	if (rcStrict == VINF_SUCCESS)
6322	return pvMem;
6323	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6324	}
6325
6326	#ifdef IEM_WITH_DATA_TLB
6327	Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
6328
6329	/*
6330	* Get the TLB entry for this page.
6331	*/
6332	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrMem);
6333	PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6334	if (pTlbe->uTag == uTag)
6335	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6336	else
6337	{
6338	pVCpu->iem.s.DataTlb.cTlbMisses++;
6339	PGMPTWALK Walk;
6340	int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
6341	if (RT_FAILURE(rc))
6342	{
6343	Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
6344	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6345	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6346	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
6347	# endif
6348	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, rc);
6349	}
6350
6351	Assert(Walk.fSucceeded);
6352	pTlbe->uTag = uTag;
6353	pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US \| X86_PTE_RW \| X86_PTE_D \| X86_PTE_A); /* skipping NX */
6354	pTlbe->GCPhys = Walk.GCPhys;
6355	pTlbe->pbMappingR3 = NULL;
6356	}
6357
6358	/*
6359	* Check the flags and physical revision.
6360	*/
6361	/** @todo make the caller pass these in with fAccess. */
6362	uint64_t const fNoUser = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && pVCpu->iem.s.uCpl == 3
6363	? IEMTLBE_F_PT_NO_USER : 0;
6364	uint64_t const fNoWriteNoDirty = fAccess & IEM_ACCESS_TYPE_WRITE
6365	? IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY
6366	\| ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)
6367	\|\| (pVCpu->iem.s.uCpl == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
6368	? IEMTLBE_F_PT_NO_WRITE : 0)
6369	: 0;
6370	uint64_t const fNoRead = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0;
6371	uint8_t *pbMem = NULL;
6372	if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PT_NO_ACCESSED \| fNoRead \| fNoWriteNoDirty \| fNoUser))
6373	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6374	# ifdef IN_RING3
6375	pbMem = pTlbe->pbMappingR3;
6376	# else
6377	pbMem = NULL;
6378	# endif
6379	else
6380	{
6381	/*
6382	* Okay, something isn't quite right or needs refreshing.
6383	*/
6384	/* Write to read only memory? */
6385	if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
6386	{
6387	Log(("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
6388	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6389	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6390	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6391	# endif
6392	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
6393	}
6394
6395	/* Kernel memory accessed by userland? */
6396	if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER)
6397	{
6398	Log(("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
6399	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
6400	if (Walk.fFailed & PGM_WALKFAIL_EPT)
6401	IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
6402	# endif
6403	iemRaisePageFaultJmp(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
6404	}
6405
6406	/* Set the dirty / access flags.
6407	ASSUMES this is set when the address is translated rather than on commit... */
6408	/** @todo testcase: check when A and D bits are actually set by the CPU. */
6409	if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED))
6410	{
6411	uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D \| X86_PTE_A : X86_PTE_A;
6412	int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
6413	AssertRC(rc2);
6414	/** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
6415	Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
6416	pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) \| IEMTLBE_F_PT_NO_ACCESSED);
6417	}
6418
6419	/*
6420	* Check if the physical page info needs updating.
6421	*/
6422	if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
6423	# ifdef IN_RING3
6424	pbMem = pTlbe->pbMappingR3;
6425	# else
6426	pbMem = NULL;
6427	# endif
6428	else
6429	{
6430	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
6431	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
6432	AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
6433	AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED == IEMTLBE_F_PG_UNASSIGNED);
6434	pTlbe->pbMappingR3 = NULL;
6435	pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
6436	\| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_NO_READ \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_UNASSIGNED);
6437	int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
6438	&pbMem, &pTlbe->fFlagsAndPhysRev);
6439	AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
6440	# ifdef IN_RING3
6441	pTlbe->pbMappingR3 = pbMem;
6442	# endif
6443	}
6444
6445	/*
6446	* Check the physical page level access and mapping.
6447	*/
6448	if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty \| fNoRead) & (IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PG_NO_READ))))
6449	{ /* probably likely */ }
6450	else
6451	{
6452	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, cbMem,
6453	pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
6454	pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
6455	: pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ ? VERR_PGM_PHYS_TLB_CATCH_ALL
6456	: VERR_PGM_PHYS_TLB_CATCH_WRITE);
6457	if (rcStrict == VINF_SUCCESS)
6458	return pbMem;
6459	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6460	}
6461	}
6462	Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
6463
6464	if (pbMem)
6465	{
6466	Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
6467	pbMem = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6468	fAccess \|= IEM_ACCESS_NOT_LOCKED;
6469	}
6470	else
6471	{
6472	Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
6473	RTGCPHYS const GCPhysFirst = pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
6474	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6475	if (rcStrict == VINF_SUCCESS)
6476	return pbMem;
6477	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6478	}
6479
6480	void * const pvMem = pbMem;
6481
6482	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6483	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6484	if (fAccess & IEM_ACCESS_TYPE_READ)
6485	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys \| (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
6486
6487	#else /* !IEM_WITH_DATA_TLB */
6488
6489
6490	RTGCPHYS GCPhysFirst;
6491	rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
6492	if (rcStrict == VINF_SUCCESS) { /likely/ }
6493	else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6494
6495	if (fAccess & IEM_ACCESS_TYPE_WRITE)
6496	Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6497	if (fAccess & IEM_ACCESS_TYPE_READ)
6498	Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
6499
6500	void *pvMem;
6501	rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6502	if (rcStrict == VINF_SUCCESS)
6503	{ /* likely */ }
6504	else
6505	{
6506	rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
6507	if (rcStrict == VINF_SUCCESS)
6508	return pvMem;
6509	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6510	}
6511
6512	#endif /* !IEM_WITH_DATA_TLB */
6513
6514	/*
6515	* Fill in the mapping table entry.
6516	*/
6517	pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
6518	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
6519	pVCpu->iem.s.iNextMapping = iMemMap + 1;
6520	pVCpu->iem.s.cActiveMappings++;
6521
6522	iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
6523	return pvMem;
6524	}
6525
6526
6527	/**
6528	* Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
6529	*
6530	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6531	* @param pvMem The mapping.
6532	* @param fAccess The kind of access.
6533	*/
6534	void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6535	{
6536	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6537	AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
6538
6539	/* If it's bounce buffered, we may need to write back the buffer. */
6540	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6541	{
6542	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6543	{
6544	VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /fPostponeFail/);
6545	if (rcStrict == VINF_SUCCESS)
6546	return;
6547	longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
6548	}
6549	}
6550	/* Otherwise unlock it. */
6551	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6552	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6553
6554	/* Free the entry. */
6555	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6556	Assert(pVCpu->iem.s.cActiveMappings != 0);
6557	pVCpu->iem.s.cActiveMappings--;
6558	}
6559
6560	#endif /* IEM_WITH_SETJMP */
6561
6562	#ifndef IN_RING3
6563	/**
6564	* Commits the guest memory if bounce buffered and unmaps it, if any bounce
6565	* buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
6566	*
6567	* Allows the instruction to be completed and retired, while the IEM user will
6568	* return to ring-3 immediately afterwards and do the postponed writes there.
6569	*
6570	* @returns VBox status code (no strict statuses). Caller must check
6571	* VMCPU_FF_IEM before repeating string instructions and similar stuff.
6572	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6573	* @param pvMem The mapping.
6574	* @param fAccess The kind of access.
6575	*/
6576	VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
6577	{
6578	int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
6579	AssertReturn(iMemMap >= 0, iMemMap);
6580
6581	/* If it's bounce buffered, we may need to write back the buffer. */
6582	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
6583	{
6584	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
6585	return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /fPostponeFail/);
6586	}
6587	/* Otherwise unlock it. */
6588	else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
6589	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6590
6591	/* Free the entry. */
6592	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6593	Assert(pVCpu->iem.s.cActiveMappings != 0);
6594	pVCpu->iem.s.cActiveMappings--;
6595	return VINF_SUCCESS;
6596	}
6597	#endif
6598
6599
6600	/**
6601	* Rollbacks mappings, releasing page locks and such.
6602	*
6603	* The caller shall only call this after checking cActiveMappings.
6604	*
6605	* @returns Strict VBox status code to pass up.
6606	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6607	*/
6608	void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT
6609	{
6610	Assert(pVCpu->iem.s.cActiveMappings > 0);
6611
6612	uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
6613	while (iMemMap-- > 0)
6614	{
6615	uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
6616	if (fAccess != IEM_ACCESS_INVALID)
6617	{
6618	AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
6619	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
6620	if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED \| IEM_ACCESS_NOT_LOCKED)))
6621	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
6622	AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
6623	("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
6624	iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
6625	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
6626	pVCpu->iem.s.cActiveMappings--;
6627	}
6628	}
6629	}
6630
6631
6632	/**
6633	* Fetches a data byte.
6634	*
6635	* @returns Strict VBox status code.
6636	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6637	* @param pu8Dst Where to return the byte.
6638	* @param iSegReg The index of the segment register to use for
6639	* this access. The base and limits are checked.
6640	* @param GCPtrMem The address of the guest memory.
6641	*/
6642	VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6643	{
6644	/* The lazy approach for now... */
6645	uint8_t const *pu8Src;
6646	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu8Src, sizeof(pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6647	if (rc == VINF_SUCCESS)
6648	{
6649	pu8Dst = pu8Src;
6650	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6651	}
6652	return rc;
6653	}
6654
6655
6656	#ifdef IEM_WITH_SETJMP
6657	/**
6658	* Fetches a data byte, longjmp on error.
6659	*
6660	* @returns The byte.
6661	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6662	* @param iSegReg The index of the segment register to use for
6663	* this access. The base and limits are checked.
6664	* @param GCPtrMem The address of the guest memory.
6665	*/
6666	uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6667	{
6668	/* The lazy approach for now... */
6669	uint8_t const pu8Src = (uint8_t const )iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
6670	uint8_t const bRet = *pu8Src;
6671	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
6672	return bRet;
6673	}
6674	#endif /* IEM_WITH_SETJMP */
6675
6676
6677	/**
6678	* Fetches a data word.
6679	*
6680	* @returns Strict VBox status code.
6681	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6682	* @param pu16Dst Where to return the word.
6683	* @param iSegReg The index of the segment register to use for
6684	* this access. The base and limits are checked.
6685	* @param GCPtrMem The address of the guest memory.
6686	*/
6687	VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6688	{
6689	/* The lazy approach for now... */
6690	uint16_t const *pu16Src;
6691	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem,
6692	IEM_ACCESS_DATA_R, sizeof(*pu16Src) - 1);
6693	if (rc == VINF_SUCCESS)
6694	{
6695	pu16Dst = pu16Src;
6696	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6697	}
6698	return rc;
6699	}
6700
6701
6702	#ifdef IEM_WITH_SETJMP
6703	/**
6704	* Fetches a data word, longjmp on error.
6705	*
6706	* @returns The word
6707	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6708	* @param iSegReg The index of the segment register to use for
6709	* this access. The base and limits are checked.
6710	* @param GCPtrMem The address of the guest memory.
6711	*/
6712	uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6713	{
6714	/* The lazy approach for now... */
6715	uint16_t const pu16Src = (uint16_t const )iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6716	sizeof(*pu16Src) - 1);
6717	uint16_t const u16Ret = *pu16Src;
6718	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
6719	return u16Ret;
6720	}
6721	#endif
6722
6723
6724	/**
6725	* Fetches a data dword.
6726	*
6727	* @returns Strict VBox status code.
6728	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6729	* @param pu32Dst Where to return the dword.
6730	* @param iSegReg The index of the segment register to use for
6731	* this access. The base and limits are checked.
6732	* @param GCPtrMem The address of the guest memory.
6733	*/
6734	VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6735	{
6736	/* The lazy approach for now... */
6737	uint32_t const *pu32Src;
6738	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem,
6739	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6740	if (rc == VINF_SUCCESS)
6741	{
6742	pu32Dst = pu32Src;
6743	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6744	}
6745	return rc;
6746	}
6747
6748
6749	/**
6750	* Fetches a data dword and zero extends it to a qword.
6751	*
6752	* @returns Strict VBox status code.
6753	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6754	* @param pu64Dst Where to return the qword.
6755	* @param iSegReg The index of the segment register to use for
6756	* this access. The base and limits are checked.
6757	* @param GCPtrMem The address of the guest memory.
6758	*/
6759	VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6760	{
6761	/* The lazy approach for now... */
6762	uint32_t const *pu32Src;
6763	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem,
6764	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6765	if (rc == VINF_SUCCESS)
6766	{
6767	pu64Dst = pu32Src;
6768	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6769	}
6770	return rc;
6771	}
6772
6773
6774	#ifdef IEM_WITH_SETJMP
6775
6776	/**
6777	* Fetches a data dword, longjmp on error, fallback/safe version.
6778	*
6779	* @returns The dword
6780	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6781	* @param iSegReg The index of the segment register to use for
6782	* this access. The base and limits are checked.
6783	* @param GCPtrMem The address of the guest memory.
6784	*/
6785	uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6786	{
6787	uint32_t const pu32Src = (uint32_t const )iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6788	sizeof(*pu32Src) - 1);
6789	uint32_t const u32Ret = *pu32Src;
6790	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6791	return u32Ret;
6792	}
6793
6794
6795	/**
6796	* Fetches a data dword, longjmp on error.
6797	*
6798	* @returns The dword
6799	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6800	* @param iSegReg The index of the segment register to use for
6801	* this access. The base and limits are checked.
6802	* @param GCPtrMem The address of the guest memory.
6803	*/
6804	uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6805	{
6806	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
6807	/*
6808	* Convert from segmented to flat address and check that it doesn't cross a page boundrary.
6809	*/
6810	RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
6811	if (RT_LIKELY((GCPtrEff & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(uint32_t)))
6812	{
6813	/*
6814	* TLB lookup.
6815	*/
6816	uint64_t const uTag = IEMTLB_CALC_TAG( &pVCpu->iem.s.DataTlb, GCPtrEff);
6817	PIEMTLBENTRY pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
6818	if (pTlbe->uTag == uTag)
6819	{
6820	/*
6821	* Check TLB page table level access flags.
6822	*/
6823	uint64_t const fNoUser = pVCpu->iem.s.uCpl == 3 ? IEMTLBE_F_PT_NO_USER : 0;
6824	if ( (pTlbe->fFlagsAndPhysRev & ( IEMTLBE_F_PHYS_REV \| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PG_NO_READ
6825	\| IEMTLBE_F_PT_NO_ACCESSED \| IEMTLBE_F_NO_MAPPINGR3 \| fNoUser))
6826	== pVCpu->iem.s.DataTlb.uTlbPhysRev)
6827	{
6828	STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
6829
6830	/*
6831	* Alignment check:
6832	*/
6833	/** @todo check priority \#AC vs \#PF */
6834	if ( !(GCPtrEff & (sizeof(uint32_t) - 1))
6835	\|\| !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
6836	\|\| !pVCpu->cpum.GstCtx.eflags.Bits.u1AC
6837	\|\| pVCpu->iem.s.uCpl != 3)
6838	{
6839	/*
6840	* Fetch and return the dword
6841	*/
6842	Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
6843	Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
6844	return (uint32_t const )&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
6845	}
6846	Log10(("iemMemFetchDataU32Jmp: Raising #AC for %RGv\n", GCPtrEff));
6847	iemRaiseAlignmentCheckExceptionJmp(pVCpu);
6848	}
6849	}
6850	}
6851
6852	/* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
6853	outdated page pointer, or other troubles. */
6854	Log10(("iemMemFetchDataU32Jmp: %u:%RGv fallback\n", iSegReg, GCPtrMem));
6855	return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
6856
6857	# else
6858	uint32_t const pu32Src = (uint32_t const )iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem,
6859	IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
6860	uint32_t const u32Ret = *pu32Src;
6861	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
6862	return u32Ret;
6863	# endif
6864	}
6865	#endif
6866
6867
6868	#ifdef SOME_UNUSED_FUNCTION
6869	/**
6870	* Fetches a data dword and sign extends it to a qword.
6871	*
6872	* @returns Strict VBox status code.
6873	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6874	* @param pu64Dst Where to return the sign extended value.
6875	* @param iSegReg The index of the segment register to use for
6876	* this access. The base and limits are checked.
6877	* @param GCPtrMem The address of the guest memory.
6878	*/
6879	VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6880	{
6881	/* The lazy approach for now... */
6882	int32_t const *pi32Src;
6883	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pi32Src, sizeof(pi32Src), iSegReg, GCPtrMem,
6884	IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1);
6885	if (rc == VINF_SUCCESS)
6886	{
6887	pu64Dst = pi32Src;
6888	rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
6889	}
6890	#ifdef __GNUC__ /* warning: GCC may be a royal pain */
6891	else
6892	*pu64Dst = 0;
6893	#endif
6894	return rc;
6895	}
6896	#endif
6897
6898
6899	/**
6900	* Fetches a data qword.
6901	*
6902	* @returns Strict VBox status code.
6903	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6904	* @param pu64Dst Where to return the qword.
6905	* @param iSegReg The index of the segment register to use for
6906	* this access. The base and limits are checked.
6907	* @param GCPtrMem The address of the guest memory.
6908	*/
6909	VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6910	{
6911	/* The lazy approach for now... */
6912	uint64_t const *pu64Src;
6913	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem,
6914	IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6915	if (rc == VINF_SUCCESS)
6916	{
6917	pu64Dst = pu64Src;
6918	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6919	}
6920	return rc;
6921	}
6922
6923
6924	#ifdef IEM_WITH_SETJMP
6925	/**
6926	* Fetches a data qword, longjmp on error.
6927	*
6928	* @returns The qword.
6929	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6930	* @param iSegReg The index of the segment register to use for
6931	* this access. The base and limits are checked.
6932	* @param GCPtrMem The address of the guest memory.
6933	*/
6934	uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6935	{
6936	/* The lazy approach for now... */
6937	uint64_t const pu64Src = (uint64_t const )iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem,
6938	IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
6939	uint64_t const u64Ret = *pu64Src;
6940	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6941	return u64Ret;
6942	}
6943	#endif
6944
6945
6946	/**
6947	* Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
6948	*
6949	* @returns Strict VBox status code.
6950	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6951	* @param pu64Dst Where to return the qword.
6952	* @param iSegReg The index of the segment register to use for
6953	* this access. The base and limits are checked.
6954	* @param GCPtrMem The address of the guest memory.
6955	*/
6956	VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6957	{
6958	/* The lazy approach for now... */
6959	uint64_t const *pu64Src;
6960	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem,
6961	IEM_ACCESS_DATA_R, 15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
6962	if (rc == VINF_SUCCESS)
6963	{
6964	pu64Dst = pu64Src;
6965	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6966	}
6967	return rc;
6968	}
6969
6970
6971	#ifdef IEM_WITH_SETJMP
6972	/**
6973	* Fetches a data qword, longjmp on error.
6974	*
6975	* @returns The qword.
6976	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6977	* @param iSegReg The index of the segment register to use for
6978	* this access. The base and limits are checked.
6979	* @param GCPtrMem The address of the guest memory.
6980	*/
6981	uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
6982	{
6983	/* The lazy approach for now... */
6984	uint64_t const pu64Src = (uint64_t const )iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
6985	15 \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
6986	uint64_t const u64Ret = *pu64Src;
6987	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
6988	return u64Ret;
6989	}
6990	#endif
6991
6992
6993	/**
6994	* Fetches a data tword.
6995	*
6996	* @returns Strict VBox status code.
6997	* @param pVCpu The cross context virtual CPU structure of the calling thread.
6998	* @param pr80Dst Where to return the tword.
6999	* @param iSegReg The index of the segment register to use for
7000	* this access. The base and limits are checked.
7001	* @param GCPtrMem The address of the guest memory.
7002	*/
7003	VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7004	{
7005	/* The lazy approach for now... */
7006	PCRTFLOAT80U pr80Src;
7007	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pr80Src, sizeof(pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
7008	if (rc == VINF_SUCCESS)
7009	{
7010	pr80Dst = pr80Src;
7011	rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7012	}
7013	return rc;
7014	}
7015
7016
7017	#ifdef IEM_WITH_SETJMP
7018	/**
7019	* Fetches a data tword, longjmp on error.
7020	*
7021	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7022	* @param pr80Dst Where to return the tword.
7023	* @param iSegReg The index of the segment register to use for
7024	* this access. The base and limits are checked.
7025	* @param GCPtrMem The address of the guest memory.
7026	*/
7027	void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7028	{
7029	/* The lazy approach for now... */
7030	PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
7031	pr80Dst = pr80Src;
7032	iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
7033	}
7034	#endif
7035
7036
7037	/**
7038	* Fetches a data decimal tword.
7039	*
7040	* @returns Strict VBox status code.
7041	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7042	* @param pd80Dst Where to return the tword.
7043	* @param iSegReg The index of the segment register to use for
7044	* this access. The base and limits are checked.
7045	* @param GCPtrMem The address of the guest memory.
7046	*/
7047	VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7048	{
7049	/* The lazy approach for now... */
7050	PCRTPBCD80U pd80Src;
7051	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pd80Src, sizeof(pd80Src), iSegReg, GCPtrMem,
7052	IEM_ACCESS_DATA_R, 7 /** @todo FBLD alignment check */);
7053	if (rc == VINF_SUCCESS)
7054	{
7055	pd80Dst = pd80Src;
7056	rc = iemMemCommitAndUnmap(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7057	}
7058	return rc;
7059	}
7060
7061
7062	#ifdef IEM_WITH_SETJMP
7063	/**
7064	* Fetches a data decimal tword, longjmp on error.
7065	*
7066	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7067	* @param pd80Dst Where to return the tword.
7068	* @param iSegReg The index of the segment register to use for
7069	* this access. The base and limits are checked.
7070	* @param GCPtrMem The address of the guest memory.
7071	*/
7072	void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7073	{
7074	/* The lazy approach for now... */
7075	PCRTPBCD80U pd80Src = (PCRTPBCD80U)iemMemMapJmp(pVCpu, sizeof(*pd80Src), iSegReg, GCPtrMem,
7076	IEM_ACCESS_DATA_R, 7 /** @todo FBSTP alignment check */);
7077	pd80Dst = pd80Src;
7078	iemMemCommitAndUnmapJmp(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
7079	}
7080	#endif
7081
7082
7083	/**
7084	* Fetches a data dqword (double qword), generally SSE related.
7085	*
7086	* @returns Strict VBox status code.
7087	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7088	* @param pu128Dst Where to return the qword.
7089	* @param iSegReg The index of the segment register to use for
7090	* this access. The base and limits are checked.
7091	* @param GCPtrMem The address of the guest memory.
7092	*/
7093	VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7094	{
7095	/* The lazy approach for now... */
7096	PCRTUINT128U pu128Src;
7097	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem,
7098	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7099	if (rc == VINF_SUCCESS)
7100	{
7101	pu128Dst->au64[0] = pu128Src->au64[0];
7102	pu128Dst->au64[1] = pu128Src->au64[1];
7103	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7104	}
7105	return rc;
7106	}
7107
7108
7109	#ifdef IEM_WITH_SETJMP
7110	/**
7111	* Fetches a data dqword (double qword), generally SSE related.
7112	*
7113	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7114	* @param pu128Dst Where to return the qword.
7115	* @param iSegReg The index of the segment register to use for
7116	* this access. The base and limits are checked.
7117	* @param GCPtrMem The address of the guest memory.
7118	*/
7119	void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7120	{
7121	/* The lazy approach for now... */
7122	PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem,
7123	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7124	pu128Dst->au64[0] = pu128Src->au64[0];
7125	pu128Dst->au64[1] = pu128Src->au64[1];
7126	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7127	}
7128	#endif
7129
7130
7131	/**
7132	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7133	* related.
7134	*
7135	* Raises \#GP(0) if not aligned.
7136	*
7137	* @returns Strict VBox status code.
7138	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7139	* @param pu128Dst Where to return the qword.
7140	* @param iSegReg The index of the segment register to use for
7141	* this access. The base and limits are checked.
7142	* @param GCPtrMem The address of the guest memory.
7143	*/
7144	VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7145	{
7146	/* The lazy approach for now... */
7147	PCRTUINT128U pu128Src;
7148	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Src, sizeof(pu128Src), iSegReg, GCPtrMem,
7149	IEM_ACCESS_DATA_R, (sizeof(*pu128Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7150	if (rc == VINF_SUCCESS)
7151	{
7152	pu128Dst->au64[0] = pu128Src->au64[0];
7153	pu128Dst->au64[1] = pu128Src->au64[1];
7154	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7155	}
7156	return rc;
7157	}
7158
7159
7160	#ifdef IEM_WITH_SETJMP
7161	/**
7162	* Fetches a data dqword (double qword) at an aligned address, generally SSE
7163	* related, longjmp on error.
7164	*
7165	* Raises \#GP(0) if not aligned.
7166	*
7167	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7168	* @param pu128Dst Where to return the qword.
7169	* @param iSegReg The index of the segment register to use for
7170	* this access. The base and limits are checked.
7171	* @param GCPtrMem The address of the guest memory.
7172	*/
7173	void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7174	{
7175	/* The lazy approach for now... */
7176	PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7177	(sizeof(*pu128Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7178	pu128Dst->au64[0] = pu128Src->au64[0];
7179	pu128Dst->au64[1] = pu128Src->au64[1];
7180	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
7181	}
7182	#endif
7183
7184
7185	/**
7186	* Fetches a data oword (octo word), generally AVX related.
7187	*
7188	* @returns Strict VBox status code.
7189	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7190	* @param pu256Dst Where to return the qword.
7191	* @param iSegReg The index of the segment register to use for
7192	* this access. The base and limits are checked.
7193	* @param GCPtrMem The address of the guest memory.
7194	*/
7195	VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7196	{
7197	/* The lazy approach for now... */
7198	PCRTUINT256U pu256Src;
7199	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, sizeof(pu256Src), iSegReg, GCPtrMem,
7200	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7201	if (rc == VINF_SUCCESS)
7202	{
7203	pu256Dst->au64[0] = pu256Src->au64[0];
7204	pu256Dst->au64[1] = pu256Src->au64[1];
7205	pu256Dst->au64[2] = pu256Src->au64[2];
7206	pu256Dst->au64[3] = pu256Src->au64[3];
7207	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7208	}
7209	return rc;
7210	}
7211
7212
7213	#ifdef IEM_WITH_SETJMP
7214	/**
7215	* Fetches a data oword (octo word), generally AVX related.
7216	*
7217	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7218	* @param pu256Dst Where to return the qword.
7219	* @param iSegReg The index of the segment register to use for
7220	* this access. The base and limits are checked.
7221	* @param GCPtrMem The address of the guest memory.
7222	*/
7223	void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7224	{
7225	/* The lazy approach for now... */
7226	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem,
7227	IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
7228	pu256Dst->au64[0] = pu256Src->au64[0];
7229	pu256Dst->au64[1] = pu256Src->au64[1];
7230	pu256Dst->au64[2] = pu256Src->au64[2];
7231	pu256Dst->au64[3] = pu256Src->au64[3];
7232	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7233	}
7234	#endif
7235
7236
7237	/**
7238	* Fetches a data oword (octo word) at an aligned address, generally AVX
7239	* related.
7240	*
7241	* Raises \#GP(0) if not aligned.
7242	*
7243	* @returns Strict VBox status code.
7244	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7245	* @param pu256Dst Where to return the qword.
7246	* @param iSegReg The index of the segment register to use for
7247	* this access. The base and limits are checked.
7248	* @param GCPtrMem The address of the guest memory.
7249	*/
7250	VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7251	{
7252	/* The lazy approach for now... */
7253	PCRTUINT256U pu256Src;
7254	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Src, sizeof(pu256Src), iSegReg, GCPtrMem,
7255	IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7256	if (rc == VINF_SUCCESS)
7257	{
7258	pu256Dst->au64[0] = pu256Src->au64[0];
7259	pu256Dst->au64[1] = pu256Src->au64[1];
7260	pu256Dst->au64[2] = pu256Src->au64[2];
7261	pu256Dst->au64[3] = pu256Src->au64[3];
7262	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7263	}
7264	return rc;
7265	}
7266
7267
7268	#ifdef IEM_WITH_SETJMP
7269	/**
7270	* Fetches a data oword (octo word) at an aligned address, generally AVX
7271	* related, longjmp on error.
7272	*
7273	* Raises \#GP(0) if not aligned.
7274	*
7275	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7276	* @param pu256Dst Where to return the qword.
7277	* @param iSegReg The index of the segment register to use for
7278	* this access. The base and limits are checked.
7279	* @param GCPtrMem The address of the guest memory.
7280	*/
7281	void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7282	{
7283	/* The lazy approach for now... */
7284	PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
7285	(sizeof(*pu256Src) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7286	pu256Dst->au64[0] = pu256Src->au64[0];
7287	pu256Dst->au64[1] = pu256Src->au64[1];
7288	pu256Dst->au64[2] = pu256Src->au64[2];
7289	pu256Dst->au64[3] = pu256Src->au64[3];
7290	iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
7291	}
7292	#endif
7293
7294
7295
7296	/**
7297	* Fetches a descriptor register (lgdt, lidt).
7298	*
7299	* @returns Strict VBox status code.
7300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7301	* @param pcbLimit Where to return the limit.
7302	* @param pGCPtrBase Where to return the base.
7303	* @param iSegReg The index of the segment register to use for
7304	* this access. The base and limits are checked.
7305	* @param GCPtrMem The address of the guest memory.
7306	* @param enmOpSize The effective operand size.
7307	*/
7308	VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
7309	RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT
7310	{
7311	/*
7312	* Just like SIDT and SGDT, the LIDT and LGDT instructions are a
7313	* little special:
7314	* - The two reads are done separately.
7315	* - Operand size override works in 16-bit and 32-bit code, but 64-bit.
7316	* - We suspect the 386 to actually commit the limit before the base in
7317	* some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
7318	* don't try emulate this eccentric behavior, because it's not well
7319	* enough understood and rather hard to trigger.
7320	* - The 486 seems to do a dword limit read when the operand size is 32-bit.
7321	*/
7322	VBOXSTRICTRC rcStrict;
7323	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7324	{
7325	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7326	if (rcStrict == VINF_SUCCESS)
7327	rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
7328	}
7329	else
7330	{
7331	uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
7332	if (enmOpSize == IEMMODE_32BIT)
7333	{
7334	if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
7335	{
7336	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7337	if (rcStrict == VINF_SUCCESS)
7338	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7339	}
7340	else
7341	{
7342	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
7343	if (rcStrict == VINF_SUCCESS)
7344	{
7345	*pcbLimit = (uint16_t)uTmp;
7346	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7347	}
7348	}
7349	if (rcStrict == VINF_SUCCESS)
7350	*pGCPtrBase = uTmp;
7351	}
7352	else
7353	{
7354	rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
7355	if (rcStrict == VINF_SUCCESS)
7356	{
7357	rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
7358	if (rcStrict == VINF_SUCCESS)
7359	*pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
7360	}
7361	}
7362	}
7363	return rcStrict;
7364	}
7365
7366
7367
7368	/**
7369	* Stores a data byte.
7370	*
7371	* @returns Strict VBox status code.
7372	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7373	* @param iSegReg The index of the segment register to use for
7374	* this access. The base and limits are checked.
7375	* @param GCPtrMem The address of the guest memory.
7376	* @param u8Value The value to store.
7377	*/
7378	VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7379	{
7380	/* The lazy approach for now... */
7381	uint8_t *pu8Dst;
7382	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu8Dst, sizeof(pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7383	if (rc == VINF_SUCCESS)
7384	{
7385	*pu8Dst = u8Value;
7386	rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7387	}
7388	return rc;
7389	}
7390
7391
7392	#ifdef IEM_WITH_SETJMP
7393	/**
7394	* Stores a data byte, longjmp on error.
7395	*
7396	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7397	* @param iSegReg The index of the segment register to use for
7398	* this access. The base and limits are checked.
7399	* @param GCPtrMem The address of the guest memory.
7400	* @param u8Value The value to store.
7401	*/
7402	void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
7403	{
7404	/* The lazy approach for now... */
7405	uint8_t pu8Dst = (uint8_t )iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
7406	*pu8Dst = u8Value;
7407	iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
7408	}
7409	#endif
7410
7411
7412	/**
7413	* Stores a data word.
7414	*
7415	* @returns Strict VBox status code.
7416	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7417	* @param iSegReg The index of the segment register to use for
7418	* this access. The base and limits are checked.
7419	* @param GCPtrMem The address of the guest memory.
7420	* @param u16Value The value to store.
7421	*/
7422	VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7423	{
7424	/* The lazy approach for now... */
7425	uint16_t *pu16Dst;
7426	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), iSegReg, GCPtrMem,
7427	IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7428	if (rc == VINF_SUCCESS)
7429	{
7430	*pu16Dst = u16Value;
7431	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7432	}
7433	return rc;
7434	}
7435
7436
7437	#ifdef IEM_WITH_SETJMP
7438	/**
7439	* Stores a data word, longjmp on error.
7440	*
7441	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7442	* @param iSegReg The index of the segment register to use for
7443	* this access. The base and limits are checked.
7444	* @param GCPtrMem The address of the guest memory.
7445	* @param u16Value The value to store.
7446	*/
7447	void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
7448	{
7449	/* The lazy approach for now... */
7450	uint16_t pu16Dst = (uint16_t )iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem,
7451	IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
7452	*pu16Dst = u16Value;
7453	iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
7454	}
7455	#endif
7456
7457
7458	/**
7459	* Stores a data dword.
7460	*
7461	* @returns Strict VBox status code.
7462	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7463	* @param iSegReg The index of the segment register to use for
7464	* this access. The base and limits are checked.
7465	* @param GCPtrMem The address of the guest memory.
7466	* @param u32Value The value to store.
7467	*/
7468	VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7469	{
7470	/* The lazy approach for now... */
7471	uint32_t *pu32Dst;
7472	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), iSegReg, GCPtrMem,
7473	IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7474	if (rc == VINF_SUCCESS)
7475	{
7476	*pu32Dst = u32Value;
7477	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7478	}
7479	return rc;
7480	}
7481
7482
7483	#ifdef IEM_WITH_SETJMP
7484	/**
7485	* Stores a data dword.
7486	*
7487	* @returns Strict VBox status code.
7488	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7489	* @param iSegReg The index of the segment register to use for
7490	* this access. The base and limits are checked.
7491	* @param GCPtrMem The address of the guest memory.
7492	* @param u32Value The value to store.
7493	*/
7494	void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
7495	{
7496	/* The lazy approach for now... */
7497	uint32_t pu32Dst = (uint32_t )iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem,
7498	IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
7499	*pu32Dst = u32Value;
7500	iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
7501	}
7502	#endif
7503
7504
7505	/**
7506	* Stores a data qword.
7507	*
7508	* @returns Strict VBox status code.
7509	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7510	* @param iSegReg The index of the segment register to use for
7511	* this access. The base and limits are checked.
7512	* @param GCPtrMem The address of the guest memory.
7513	* @param u64Value The value to store.
7514	*/
7515	VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7516	{
7517	/* The lazy approach for now... */
7518	uint64_t *pu64Dst;
7519	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), iSegReg, GCPtrMem,
7520	IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7521	if (rc == VINF_SUCCESS)
7522	{
7523	*pu64Dst = u64Value;
7524	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7525	}
7526	return rc;
7527	}
7528
7529
7530	#ifdef IEM_WITH_SETJMP
7531	/**
7532	* Stores a data qword, longjmp on error.
7533	*
7534	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7535	* @param iSegReg The index of the segment register to use for
7536	* this access. The base and limits are checked.
7537	* @param GCPtrMem The address of the guest memory.
7538	* @param u64Value The value to store.
7539	*/
7540	void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
7541	{
7542	/* The lazy approach for now... */
7543	uint64_t pu64Dst = (uint64_t )iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem,
7544	IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
7545	*pu64Dst = u64Value;
7546	iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
7547	}
7548	#endif
7549
7550
7551	/**
7552	* Stores a data dqword.
7553	*
7554	* @returns Strict VBox status code.
7555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7556	* @param iSegReg The index of the segment register to use for
7557	* this access. The base and limits are checked.
7558	* @param GCPtrMem The address of the guest memory.
7559	* @param u128Value The value to store.
7560	*/
7561	VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7562	{
7563	/* The lazy approach for now... */
7564	PRTUINT128U pu128Dst;
7565	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem,
7566	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7567	if (rc == VINF_SUCCESS)
7568	{
7569	pu128Dst->au64[0] = u128Value.au64[0];
7570	pu128Dst->au64[1] = u128Value.au64[1];
7571	rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7572	}
7573	return rc;
7574	}
7575
7576
7577	#ifdef IEM_WITH_SETJMP
7578	/**
7579	* Stores a data dqword, longjmp on error.
7580	*
7581	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7582	* @param iSegReg The index of the segment register to use for
7583	* this access. The base and limits are checked.
7584	* @param GCPtrMem The address of the guest memory.
7585	* @param u128Value The value to store.
7586	*/
7587	void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7588	{
7589	/* The lazy approach for now... */
7590	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem,
7591	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7592	pu128Dst->au64[0] = u128Value.au64[0];
7593	pu128Dst->au64[1] = u128Value.au64[1];
7594	iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7595	}
7596	#endif
7597
7598
7599	/**
7600	* Stores a data dqword, SSE aligned.
7601	*
7602	* @returns Strict VBox status code.
7603	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7604	* @param iSegReg The index of the segment register to use for
7605	* this access. The base and limits are checked.
7606	* @param GCPtrMem The address of the guest memory.
7607	* @param u128Value The value to store.
7608	*/
7609	VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7610	{
7611	/* The lazy approach for now... */
7612	PRTUINT128U pu128Dst;
7613	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu128Dst, sizeof(pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7614	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7615	if (rc == VINF_SUCCESS)
7616	{
7617	pu128Dst->au64[0] = u128Value.au64[0];
7618	pu128Dst->au64[1] = u128Value.au64[1];
7619	rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7620	}
7621	return rc;
7622	}
7623
7624
7625	#ifdef IEM_WITH_SETJMP
7626	/**
7627	* Stores a data dqword, SSE aligned.
7628	*
7629	* @returns Strict VBox status code.
7630	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7631	* @param iSegReg The index of the segment register to use for
7632	* this access. The base and limits are checked.
7633	* @param GCPtrMem The address of the guest memory.
7634	* @param u128Value The value to store.
7635	*/
7636	void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
7637	{
7638	/* The lazy approach for now... */
7639	PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
7640	(sizeof(*pu128Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_SSE);
7641	pu128Dst->au64[0] = u128Value.au64[0];
7642	pu128Dst->au64[1] = u128Value.au64[1];
7643	iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
7644	}
7645	#endif
7646
7647
7648	/**
7649	* Stores a data dqword.
7650	*
7651	* @returns Strict VBox status code.
7652	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7653	* @param iSegReg The index of the segment register to use for
7654	* this access. The base and limits are checked.
7655	* @param GCPtrMem The address of the guest memory.
7656	* @param pu256Value Pointer to the value to store.
7657	*/
7658	VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7659	{
7660	/* The lazy approach for now... */
7661	PRTUINT256U pu256Dst;
7662	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, sizeof(pu256Dst), iSegReg, GCPtrMem,
7663	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7664	if (rc == VINF_SUCCESS)
7665	{
7666	pu256Dst->au64[0] = pu256Value->au64[0];
7667	pu256Dst->au64[1] = pu256Value->au64[1];
7668	pu256Dst->au64[2] = pu256Value->au64[2];
7669	pu256Dst->au64[3] = pu256Value->au64[3];
7670	rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7671	}
7672	return rc;
7673	}
7674
7675
7676	#ifdef IEM_WITH_SETJMP
7677	/**
7678	* Stores a data dqword, longjmp on error.
7679	*
7680	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7681	* @param iSegReg The index of the segment register to use for
7682	* this access. The base and limits are checked.
7683	* @param GCPtrMem The address of the guest memory.
7684	* @param pu256Value Pointer to the value to store.
7685	*/
7686	void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7687	{
7688	/* The lazy approach for now... */
7689	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7690	IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
7691	pu256Dst->au64[0] = pu256Value->au64[0];
7692	pu256Dst->au64[1] = pu256Value->au64[1];
7693	pu256Dst->au64[2] = pu256Value->au64[2];
7694	pu256Dst->au64[3] = pu256Value->au64[3];
7695	iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7696	}
7697	#endif
7698
7699
7700	/**
7701	* Stores a data dqword, AVX \#GP(0) aligned.
7702	*
7703	* @returns Strict VBox status code.
7704	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7705	* @param iSegReg The index of the segment register to use for
7706	* this access. The base and limits are checked.
7707	* @param GCPtrMem The address of the guest memory.
7708	* @param pu256Value Pointer to the value to store.
7709	*/
7710	VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7711	{
7712	/* The lazy approach for now... */
7713	PRTUINT256U pu256Dst;
7714	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu256Dst, sizeof(pu256Dst), iSegReg, GCPtrMem,
7715	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7716	if (rc == VINF_SUCCESS)
7717	{
7718	pu256Dst->au64[0] = pu256Value->au64[0];
7719	pu256Dst->au64[1] = pu256Value->au64[1];
7720	pu256Dst->au64[2] = pu256Value->au64[2];
7721	pu256Dst->au64[3] = pu256Value->au64[3];
7722	rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7723	}
7724	return rc;
7725	}
7726
7727
7728	#ifdef IEM_WITH_SETJMP
7729	/**
7730	* Stores a data dqword, AVX aligned.
7731	*
7732	* @returns Strict VBox status code.
7733	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7734	* @param iSegReg The index of the segment register to use for
7735	* this access. The base and limits are checked.
7736	* @param GCPtrMem The address of the guest memory.
7737	* @param pu256Value Pointer to the value to store.
7738	*/
7739	void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
7740	{
7741	/* The lazy approach for now... */
7742	PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
7743	IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) \| IEM_MEMMAP_F_ALIGN_GP);
7744	pu256Dst->au64[0] = pu256Value->au64[0];
7745	pu256Dst->au64[1] = pu256Value->au64[1];
7746	pu256Dst->au64[2] = pu256Value->au64[2];
7747	pu256Dst->au64[3] = pu256Value->au64[3];
7748	iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
7749	}
7750	#endif
7751
7752
7753	/**
7754	* Stores a descriptor register (sgdt, sidt).
7755	*
7756	* @returns Strict VBox status code.
7757	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7758	* @param cbLimit The limit.
7759	* @param GCPtrBase The base address.
7760	* @param iSegReg The index of the segment register to use for
7761	* this access. The base and limits are checked.
7762	* @param GCPtrMem The address of the guest memory.
7763	*/
7764	VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
7765	{
7766	/*
7767	* The SIDT and SGDT instructions actually stores the data using two
7768	* independent writes (see bs3CpuBasic2_sidt_sgdt_One). The instructions
7769	* does not respond to opsize prefixes.
7770	*/
7771	VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
7772	if (rcStrict == VINF_SUCCESS)
7773	{
7774	if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
7775	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
7776	IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
7777	? (uint32_t)GCPtrBase \| UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
7778	else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
7779	rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
7780	else
7781	rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
7782	}
7783	return rcStrict;
7784	}
7785
7786
7787	/**
7788	* Pushes a word onto the stack.
7789	*
7790	* @returns Strict VBox status code.
7791	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7792	* @param u16Value The value to push.
7793	*/
7794	VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT
7795	{
7796	/* Increment the stack pointer. */
7797	uint64_t uNewRsp;
7798	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
7799
7800	/* Write the word the lazy way. */
7801	uint16_t *pu16Dst;
7802	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop,
7803	IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
7804	if (rc == VINF_SUCCESS)
7805	{
7806	*pu16Dst = u16Value;
7807	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
7808	}
7809
7810	/* Commit the new RSP value unless we an access handler made trouble. */
7811	if (rc == VINF_SUCCESS)
7812	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7813
7814	return rc;
7815	}
7816
7817
7818	/**
7819	* Pushes a dword onto the stack.
7820	*
7821	* @returns Strict VBox status code.
7822	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7823	* @param u32Value The value to push.
7824	*/
7825	VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7826	{
7827	/* Increment the stack pointer. */
7828	uint64_t uNewRsp;
7829	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7830
7831	/* Write the dword the lazy way. */
7832	uint32_t *pu32Dst;
7833	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop,
7834	IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
7835	if (rc == VINF_SUCCESS)
7836	{
7837	*pu32Dst = u32Value;
7838	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
7839	}
7840
7841	/* Commit the new RSP value unless we an access handler made trouble. */
7842	if (rc == VINF_SUCCESS)
7843	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7844
7845	return rc;
7846	}
7847
7848
7849	/**
7850	* Pushes a dword segment register value onto the stack.
7851	*
7852	* @returns Strict VBox status code.
7853	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7854	* @param u32Value The value to push.
7855	*/
7856	VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
7857	{
7858	/* Increment the stack pointer. */
7859	uint64_t uNewRsp;
7860	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
7861
7862	/* The intel docs talks about zero extending the selector register
7863	value. My actual intel CPU here might be zero extending the value
7864	but it still only writes the lower word... */
7865	/** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
7866	* happens when crossing an electric page boundrary, is the high word checked
7867	* for write accessibility or not? Probably it is. What about segment limits?
7868	* It appears this behavior is also shared with trap error codes.
7869	*
7870	* Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
7871	* ancient hardware when it actually did change. */
7872	uint16_t *pu16Dst;
7873	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop,
7874	IEM_ACCESS_STACK_RW, sizeof(pu16Dst) - 1); /* @todo 2 or 4 alignment check for PUSH SS? */
7875	if (rc == VINF_SUCCESS)
7876	{
7877	*pu16Dst = (uint16_t)u32Value;
7878	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
7879	}
7880
7881	/* Commit the new RSP value unless we an access handler made trouble. */
7882	if (rc == VINF_SUCCESS)
7883	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7884
7885	return rc;
7886	}
7887
7888
7889	/**
7890	* Pushes a qword onto the stack.
7891	*
7892	* @returns Strict VBox status code.
7893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7894	* @param u64Value The value to push.
7895	*/
7896	VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT
7897	{
7898	/* Increment the stack pointer. */
7899	uint64_t uNewRsp;
7900	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
7901
7902	/* Write the word the lazy way. */
7903	uint64_t *pu64Dst;
7904	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop,
7905	IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
7906	if (rc == VINF_SUCCESS)
7907	{
7908	*pu64Dst = u64Value;
7909	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
7910	}
7911
7912	/* Commit the new RSP value unless we an access handler made trouble. */
7913	if (rc == VINF_SUCCESS)
7914	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7915
7916	return rc;
7917	}
7918
7919
7920	/**
7921	* Pops a word from the stack.
7922	*
7923	* @returns Strict VBox status code.
7924	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7925	* @param pu16Value Where to store the popped value.
7926	*/
7927	VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT
7928	{
7929	/* Increment the stack pointer. */
7930	uint64_t uNewRsp;
7931	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
7932
7933	/* Write the word the lazy way. */
7934	uint16_t const *pu16Src;
7935	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop,
7936	IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
7937	if (rc == VINF_SUCCESS)
7938	{
7939	pu16Value = pu16Src;
7940	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
7941
7942	/* Commit the new RSP value. */
7943	if (rc == VINF_SUCCESS)
7944	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7945	}
7946
7947	return rc;
7948	}
7949
7950
7951	/**
7952	* Pops a dword from the stack.
7953	*
7954	* @returns Strict VBox status code.
7955	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7956	* @param pu32Value Where to store the popped value.
7957	*/
7958	VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT
7959	{
7960	/* Increment the stack pointer. */
7961	uint64_t uNewRsp;
7962	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
7963
7964	/* Write the word the lazy way. */
7965	uint32_t const *pu32Src;
7966	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop,
7967	IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
7968	if (rc == VINF_SUCCESS)
7969	{
7970	pu32Value = pu32Src;
7971	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
7972
7973	/* Commit the new RSP value. */
7974	if (rc == VINF_SUCCESS)
7975	pVCpu->cpum.GstCtx.rsp = uNewRsp;
7976	}
7977
7978	return rc;
7979	}
7980
7981
7982	/**
7983	* Pops a qword from the stack.
7984	*
7985	* @returns Strict VBox status code.
7986	* @param pVCpu The cross context virtual CPU structure of the calling thread.
7987	* @param pu64Value Where to store the popped value.
7988	*/
7989	VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT
7990	{
7991	/* Increment the stack pointer. */
7992	uint64_t uNewRsp;
7993	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
7994
7995	/* Write the word the lazy way. */
7996	uint64_t const *pu64Src;
7997	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop,
7998	IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
7999	if (rc == VINF_SUCCESS)
8000	{
8001	pu64Value = pu64Src;
8002	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
8003
8004	/* Commit the new RSP value. */
8005	if (rc == VINF_SUCCESS)
8006	pVCpu->cpum.GstCtx.rsp = uNewRsp;
8007	}
8008
8009	return rc;
8010	}
8011
8012
8013	/**
8014	* Pushes a word onto the stack, using a temporary stack pointer.
8015	*
8016	* @returns Strict VBox status code.
8017	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8018	* @param u16Value The value to push.
8019	* @param pTmpRsp Pointer to the temporary stack pointer.
8020	*/
8021	VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8022	{
8023	/* Increment the stack pointer. */
8024	RTUINT64U NewRsp = *pTmpRsp;
8025	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
8026
8027	/* Write the word the lazy way. */
8028	uint16_t *pu16Dst;
8029	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Dst, sizeof(pu16Dst), X86_SREG_SS, GCPtrTop,
8030	IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
8031	if (rc == VINF_SUCCESS)
8032	{
8033	*pu16Dst = u16Value;
8034	rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
8035	}
8036
8037	/* Commit the new RSP value unless we an access handler made trouble. */
8038	if (rc == VINF_SUCCESS)
8039	*pTmpRsp = NewRsp;
8040
8041	return rc;
8042	}
8043
8044
8045	/**
8046	* Pushes a dword onto the stack, using a temporary stack pointer.
8047	*
8048	* @returns Strict VBox status code.
8049	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8050	* @param u32Value The value to push.
8051	* @param pTmpRsp Pointer to the temporary stack pointer.
8052	*/
8053	VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8054	{
8055	/* Increment the stack pointer. */
8056	RTUINT64U NewRsp = *pTmpRsp;
8057	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
8058
8059	/* Write the word the lazy way. */
8060	uint32_t *pu32Dst;
8061	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Dst, sizeof(pu32Dst), X86_SREG_SS, GCPtrTop,
8062	IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
8063	if (rc == VINF_SUCCESS)
8064	{
8065	*pu32Dst = u32Value;
8066	rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
8067	}
8068
8069	/* Commit the new RSP value unless we an access handler made trouble. */
8070	if (rc == VINF_SUCCESS)
8071	*pTmpRsp = NewRsp;
8072
8073	return rc;
8074	}
8075
8076
8077	/**
8078	* Pushes a dword onto the stack, using a temporary stack pointer.
8079	*
8080	* @returns Strict VBox status code.
8081	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8082	* @param u64Value The value to push.
8083	* @param pTmpRsp Pointer to the temporary stack pointer.
8084	*/
8085	VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8086	{
8087	/* Increment the stack pointer. */
8088	RTUINT64U NewRsp = *pTmpRsp;
8089	RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
8090
8091	/* Write the word the lazy way. */
8092	uint64_t *pu64Dst;
8093	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Dst, sizeof(pu64Dst), X86_SREG_SS, GCPtrTop,
8094	IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
8095	if (rc == VINF_SUCCESS)
8096	{
8097	*pu64Dst = u64Value;
8098	rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
8099	}
8100
8101	/* Commit the new RSP value unless we an access handler made trouble. */
8102	if (rc == VINF_SUCCESS)
8103	*pTmpRsp = NewRsp;
8104
8105	return rc;
8106	}
8107
8108
8109	/**
8110	* Pops a word from the stack, using a temporary stack pointer.
8111	*
8112	* @returns Strict VBox status code.
8113	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8114	* @param pu16Value Where to store the popped value.
8115	* @param pTmpRsp Pointer to the temporary stack pointer.
8116	*/
8117	VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8118	{
8119	/* Increment the stack pointer. */
8120	RTUINT64U NewRsp = *pTmpRsp;
8121	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
8122
8123	/* Write the word the lazy way. */
8124	uint16_t const *pu16Src;
8125	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), X86_SREG_SS, GCPtrTop,
8126	IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
8127	if (rc == VINF_SUCCESS)
8128	{
8129	pu16Value = pu16Src;
8130	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
8131
8132	/* Commit the new RSP value. */
8133	if (rc == VINF_SUCCESS)
8134	*pTmpRsp = NewRsp;
8135	}
8136
8137	return rc;
8138	}
8139
8140
8141	/**
8142	* Pops a dword from the stack, using a temporary stack pointer.
8143	*
8144	* @returns Strict VBox status code.
8145	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8146	* @param pu32Value Where to store the popped value.
8147	* @param pTmpRsp Pointer to the temporary stack pointer.
8148	*/
8149	VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8150	{
8151	/* Increment the stack pointer. */
8152	RTUINT64U NewRsp = *pTmpRsp;
8153	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
8154
8155	/* Write the word the lazy way. */
8156	uint32_t const *pu32Src;
8157	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), X86_SREG_SS, GCPtrTop,
8158	IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
8159	if (rc == VINF_SUCCESS)
8160	{
8161	pu32Value = pu32Src;
8162	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
8163
8164	/* Commit the new RSP value. */
8165	if (rc == VINF_SUCCESS)
8166	*pTmpRsp = NewRsp;
8167	}
8168
8169	return rc;
8170	}
8171
8172
8173	/**
8174	* Pops a qword from the stack, using a temporary stack pointer.
8175	*
8176	* @returns Strict VBox status code.
8177	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8178	* @param pu64Value Where to store the popped value.
8179	* @param pTmpRsp Pointer to the temporary stack pointer.
8180	*/
8181	VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
8182	{
8183	/* Increment the stack pointer. */
8184	RTUINT64U NewRsp = *pTmpRsp;
8185	RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
8186
8187	/* Write the word the lazy way. */
8188	uint64_t const *pu64Src;
8189	VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), X86_SREG_SS, GCPtrTop,
8190	IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
8191	if (rcStrict == VINF_SUCCESS)
8192	{
8193	pu64Value = pu64Src;
8194	rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
8195
8196	/* Commit the new RSP value. */
8197	if (rcStrict == VINF_SUCCESS)
8198	*pTmpRsp = NewRsp;
8199	}
8200
8201	return rcStrict;
8202	}
8203
8204
8205	/**
8206	* Begin a special stack push (used by interrupt, exceptions and such).
8207	*
8208	* This will raise \#SS or \#PF if appropriate.
8209	*
8210	* @returns Strict VBox status code.
8211	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8212	* @param cbMem The number of bytes to push onto the stack.
8213	* @param cbAlign The alignment mask (7, 3, 1).
8214	* @param ppvMem Where to return the pointer to the stack memory.
8215	* As with the other memory functions this could be
8216	* direct access or bounce buffered access, so
8217	* don't commit register until the commit call
8218	* succeeds.
8219	* @param puNewRsp Where to return the new RSP value. This must be
8220	* passed unchanged to
8221	* iemMemStackPushCommitSpecial().
8222	*/
8223	VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8224	void *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8225	{
8226	Assert(cbMem < UINT8_MAX);
8227	RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
8228	return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop,
8229	IEM_ACCESS_STACK_W, cbAlign);
8230	}
8231
8232
8233	/**
8234	* Commits a special stack push (started by iemMemStackPushBeginSpecial).
8235	*
8236	* This will update the rSP.
8237	*
8238	* @returns Strict VBox status code.
8239	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8240	* @param pvMem The pointer returned by
8241	* iemMemStackPushBeginSpecial().
8242	* @param uNewRsp The new RSP value returned by
8243	* iemMemStackPushBeginSpecial().
8244	*/
8245	VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT
8246	{
8247	VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
8248	if (rcStrict == VINF_SUCCESS)
8249	pVCpu->cpum.GstCtx.rsp = uNewRsp;
8250	return rcStrict;
8251	}
8252
8253
8254	/**
8255	* Begin a special stack pop (used by iret, retf and such).
8256	*
8257	* This will raise \#SS or \#PF if appropriate.
8258	*
8259	* @returns Strict VBox status code.
8260	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8261	* @param cbMem The number of bytes to pop from the stack.
8262	* @param cbAlign The alignment mask (7, 3, 1).
8263	* @param ppvMem Where to return the pointer to the stack memory.
8264	* @param puNewRsp Where to return the new RSP value. This must be
8265	* assigned to CPUMCTX::rsp manually some time
8266	* after iemMemStackPopDoneSpecial() has been
8267	* called.
8268	*/
8269	VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
8270	void const *ppvMem, uint64_t puNewRsp) RT_NOEXCEPT
8271	{
8272	Assert(cbMem < UINT8_MAX);
8273	RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
8274	return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign);
8275	}
8276
8277
8278	/**
8279	* Continue a special stack pop (used by iret and retf), for the purpose of
8280	* retrieving a new stack pointer.
8281	*
8282	* This will raise \#SS or \#PF if appropriate.
8283	*
8284	* @returns Strict VBox status code.
8285	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8286	* @param off Offset from the top of the stack. This is zero
8287	* except in the retf case.
8288	* @param cbMem The number of bytes to pop from the stack.
8289	* @param ppvMem Where to return the pointer to the stack memory.
8290	* @param uCurNewRsp The current uncommitted RSP value. (No need to
8291	* return this because all use of this function is
8292	* to retrieve a new value and anything we return
8293	* here would be discarded.)
8294	*/
8295	VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t off, size_t cbMem,
8296	void const **ppvMem, uint64_t uCurNewRsp) RT_NOEXCEPT
8297	{
8298	Assert(cbMem < UINT8_MAX);
8299
8300	/* The essense of iemRegGetRspForPopEx and friends: / /* @todo put this into a inlined function? */
8301	RTGCPTR GCPtrTop;
8302	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
8303	GCPtrTop = uCurNewRsp;
8304	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
8305	GCPtrTop = (uint32_t)uCurNewRsp;
8306	else
8307	GCPtrTop = (uint16_t)uCurNewRsp;
8308
8309	return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop + off, IEM_ACCESS_STACK_R,
8310	0 /* checked in iemMemStackPopBeginSpecial */);
8311	}
8312
8313
8314	/**
8315	* Done with a special stack pop (started by iemMemStackPopBeginSpecial or
8316	* iemMemStackPopContinueSpecial).
8317	*
8318	* The caller will manually commit the rSP.
8319	*
8320	* @returns Strict VBox status code.
8321	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8322	* @param pvMem The pointer returned by
8323	* iemMemStackPopBeginSpecial() or
8324	* iemMemStackPopContinueSpecial().
8325	*/
8326	VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT
8327	{
8328	return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
8329	}
8330
8331
8332	/**
8333	* Fetches a system table byte.
8334	*
8335	* @returns Strict VBox status code.
8336	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8337	* @param pbDst Where to return the byte.
8338	* @param iSegReg The index of the segment register to use for
8339	* this access. The base and limits are checked.
8340	* @param GCPtrMem The address of the guest memory.
8341	*/
8342	VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8343	{
8344	/* The lazy approach for now... */
8345	uint8_t const *pbSrc;
8346	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pbSrc, sizeof(pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8347	if (rc == VINF_SUCCESS)
8348	{
8349	pbDst = pbSrc;
8350	rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
8351	}
8352	return rc;
8353	}
8354
8355
8356	/**
8357	* Fetches a system table word.
8358	*
8359	* @returns Strict VBox status code.
8360	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8361	* @param pu16Dst Where to return the word.
8362	* @param iSegReg The index of the segment register to use for
8363	* this access. The base and limits are checked.
8364	* @param GCPtrMem The address of the guest memory.
8365	*/
8366	VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8367	{
8368	/* The lazy approach for now... */
8369	uint16_t const *pu16Src;
8370	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu16Src, sizeof(pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8371	if (rc == VINF_SUCCESS)
8372	{
8373	pu16Dst = pu16Src;
8374	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
8375	}
8376	return rc;
8377	}
8378
8379
8380	/**
8381	* Fetches a system table dword.
8382	*
8383	* @returns Strict VBox status code.
8384	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8385	* @param pu32Dst Where to return the dword.
8386	* @param iSegReg The index of the segment register to use for
8387	* this access. The base and limits are checked.
8388	* @param GCPtrMem The address of the guest memory.
8389	*/
8390	VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8391	{
8392	/* The lazy approach for now... */
8393	uint32_t const *pu32Src;
8394	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu32Src, sizeof(pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8395	if (rc == VINF_SUCCESS)
8396	{
8397	pu32Dst = pu32Src;
8398	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
8399	}
8400	return rc;
8401	}
8402
8403
8404	/**
8405	* Fetches a system table qword.
8406	*
8407	* @returns Strict VBox status code.
8408	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8409	* @param pu64Dst Where to return the qword.
8410	* @param iSegReg The index of the segment register to use for
8411	* this access. The base and limits are checked.
8412	* @param GCPtrMem The address of the guest memory.
8413	*/
8414	VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
8415	{
8416	/* The lazy approach for now... */
8417	uint64_t const *pu64Src;
8418	VBOXSTRICTRC rc = iemMemMap(pVCpu, (void *)&pu64Src, sizeof(pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
8419	if (rc == VINF_SUCCESS)
8420	{
8421	pu64Dst = pu64Src;
8422	rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
8423	}
8424	return rc;
8425	}
8426
8427
8428	/**
8429	* Fetches a descriptor table entry with caller specified error code.
8430	*
8431	* @returns Strict VBox status code.
8432	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8433	* @param pDesc Where to return the descriptor table entry.
8434	* @param uSel The selector which table entry to fetch.
8435	* @param uXcpt The exception to raise on table lookup error.
8436	* @param uErrorCode The error code associated with the exception.
8437	*/
8438	static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
8439	uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT
8440	{
8441	AssertPtr(pDesc);
8442	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR \| CPUMCTX_EXTRN_LDTR);
8443
8444	/** @todo did the 286 require all 8 bytes to be accessible? */
8445	/*
8446	* Get the selector table base and check bounds.
8447	*/
8448	RTGCPTR GCPtrBase;
8449	if (uSel & X86_SEL_LDT)
8450	{
8451	if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
8452	\|\| (uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
8453	{
8454	Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
8455	uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
8456	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8457	uErrorCode, 0);
8458	}
8459
8460	Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
8461	GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
8462	}
8463	else
8464	{
8465	if ((uSel \| X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
8466	{
8467	Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
8468	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR,
8469	uErrorCode, 0);
8470	}
8471	GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
8472	}
8473
8474	/*
8475	* Read the legacy descriptor and maybe the long mode extensions if
8476	* required.
8477	*/
8478	VBOXSTRICTRC rcStrict;
8479	if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
8480	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
8481	else
8482	{
8483	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
8484	if (rcStrict == VINF_SUCCESS)
8485	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
8486	if (rcStrict == VINF_SUCCESS)
8487	rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
8488	if (rcStrict == VINF_SUCCESS)
8489	pDesc->Legacy.au16[3] = 0;
8490	else
8491	return rcStrict;
8492	}
8493
8494	if (rcStrict == VINF_SUCCESS)
8495	{
8496	if ( !IEM_IS_LONG_MODE(pVCpu)
8497	\|\| pDesc->Legacy.Gen.u1DescType)
8498	pDesc->Long.au64[1] = 0;
8499	else if ((uint32_t)(uSel \| X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
8500	rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel \| X86_SEL_RPL_LDT) + 1);
8501	else
8502	{
8503	Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
8504	/** @todo is this the right exception? */
8505	return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT \| IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
8506	}
8507	}
8508	return rcStrict;
8509	}
8510
8511
8512	/**
8513	* Fetches a descriptor table entry.
8514	*
8515	* @returns Strict VBox status code.
8516	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8517	* @param pDesc Where to return the descriptor table entry.
8518	* @param uSel The selector which table entry to fetch.
8519	* @param uXcpt The exception to raise on table lookup error.
8520	*/
8521	VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT
8522	{
8523	return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
8524	}
8525
8526
8527	/**
8528	* Marks the selector descriptor as accessed (only non-system descriptors).
8529	*
8530	* This function ASSUMES that iemMemFetchSelDesc has be called previously and
8531	* will therefore skip the limit checks.
8532	*
8533	* @returns Strict VBox status code.
8534	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8535	* @param uSel The selector.
8536	*/
8537	VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
8538	{
8539	/*
8540	* Get the selector table base and calculate the entry address.
8541	*/
8542	RTGCPTR GCPtr = uSel & X86_SEL_LDT
8543	? pVCpu->cpum.GstCtx.ldtr.u64Base
8544	: pVCpu->cpum.GstCtx.gdtr.pGdt;
8545	GCPtr += uSel & X86_SEL_MASK;
8546
8547	/*
8548	* ASMAtomicBitSet will assert if the address is misaligned, so do some
8549	* ugly stuff to avoid this. This will make sure it's an atomic access
8550	* as well more or less remove any question about 8-bit or 32-bit accesss.
8551	*/
8552	VBOXSTRICTRC rcStrict;
8553	uint32_t volatile *pu32;
8554	if ((GCPtr & 3) == 0)
8555	{
8556	/* The normal case, map the 32-bit bits around the accessed bit (40). */
8557	GCPtr += 2 + 2;
8558	rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8559	if (rcStrict != VINF_SUCCESS)
8560	return rcStrict;
8561	ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
8562	}
8563	else
8564	{
8565	/* The misaligned GDT/LDT case, map the whole thing. */
8566	rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
8567	if (rcStrict != VINF_SUCCESS)
8568	return rcStrict;
8569	switch ((uintptr_t)pu32 & 3)
8570	{
8571	case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
8572	case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
8573	case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
8574	case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
8575	}
8576	}
8577
8578	return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
8579	}
8580
8581	/** @} */
8582
8583	/** @name Opcode Helpers.
8584	* @{
8585	*/
8586
8587	/**
8588	* Calculates the effective address of a ModR/M memory operand.
8589	*
8590	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8591	*
8592	* @return Strict VBox status code.
8593	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8594	* @param bRm The ModRM byte.
8595	* @param cbImm The size of any immediate following the
8596	* effective address opcode bytes. Important for
8597	* RIP relative addressing.
8598	* @param pGCPtrEff Where to return the effective address.
8599	*/
8600	VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff) RT_NOEXCEPT
8601	{
8602	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8603	# define SET_SS_DEF() \
8604	do \
8605	{ \
8606	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8607	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8608	} while (0)
8609
8610	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8611	{
8612	/** @todo Check the effective address size crap! */
8613	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8614	{
8615	uint16_t u16EffAddr;
8616
8617	/* Handle the disp16 form with no registers first. */
8618	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8619	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8620	else
8621	{
8622	/* Get the displacment. */
8623	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8624	{
8625	case 0: u16EffAddr = 0; break;
8626	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8627	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8628	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8629	}
8630
8631	/* Add the base and index registers to the disp. */
8632	switch (bRm & X86_MODRM_RM_MASK)
8633	{
8634	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8635	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8636	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8637	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8638	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8639	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8640	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8641	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8642	}
8643	}
8644
8645	*pGCPtrEff = u16EffAddr;
8646	}
8647	else
8648	{
8649	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8650	uint32_t u32EffAddr;
8651
8652	/* Handle the disp32 form with no registers first. */
8653	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8654	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8655	else
8656	{
8657	/* Get the register (or SIB) value. */
8658	switch ((bRm & X86_MODRM_RM_MASK))
8659	{
8660	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8661	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8662	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8663	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8664	case 4: /* SIB */
8665	{
8666	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8667
8668	/* Get the index and scale it. */
8669	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8670	{
8671	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8672	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8673	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8674	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8675	case 4: u32EffAddr = 0; /none / break;
8676	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8677	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8678	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8679	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8680	}
8681	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8682
8683	/* add base */
8684	switch (bSib & X86_SIB_BASE_MASK)
8685	{
8686	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8687	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8688	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8689	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8690	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
8691	case 5:
8692	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8693	{
8694	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8695	SET_SS_DEF();
8696	}
8697	else
8698	{
8699	uint32_t u32Disp;
8700	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8701	u32EffAddr += u32Disp;
8702	}
8703	break;
8704	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
8705	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
8706	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8707	}
8708	break;
8709	}
8710	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
8711	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8712	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8713	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8714	}
8715
8716	/* Get and add the displacement. */
8717	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8718	{
8719	case 0:
8720	break;
8721	case 1:
8722	{
8723	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8724	u32EffAddr += i8Disp;
8725	break;
8726	}
8727	case 2:
8728	{
8729	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8730	u32EffAddr += u32Disp;
8731	break;
8732	}
8733	default:
8734	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
8735	}
8736
8737	}
8738	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
8739	*pGCPtrEff = u32EffAddr;
8740	else
8741	{
8742	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
8743	*pGCPtrEff = u32EffAddr & UINT16_MAX;
8744	}
8745	}
8746	}
8747	else
8748	{
8749	uint64_t u64EffAddr;
8750
8751	/* Handle the rip+disp32 form with no registers first. */
8752	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8753	{
8754	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
8755	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
8756	}
8757	else
8758	{
8759	/* Get the register (or SIB) value. */
8760	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
8761	{
8762	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8763	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8764	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8765	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8766	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
8767	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8768	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8769	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8770	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8771	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8772	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8773	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8774	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8775	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8776	/* SIB */
8777	case 4:
8778	case 12:
8779	{
8780	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8781
8782	/* Get the index and scale it. */
8783	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
8784	{
8785	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
8786	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
8787	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
8788	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
8789	case 4: u64EffAddr = 0; /none / break;
8790	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
8791	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
8792	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
8793	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
8794	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
8795	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
8796	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
8797	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
8798	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
8799	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
8800	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
8801	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8802	}
8803	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8804
8805	/* add base */
8806	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
8807	{
8808	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
8809	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
8810	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
8811	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
8812	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
8813	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
8814	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
8815	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
8816	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
8817	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
8818	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
8819	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
8820	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
8821	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
8822	/* complicated encodings */
8823	case 5:
8824	case 13:
8825	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8826	{
8827	if (!pVCpu->iem.s.uRexB)
8828	{
8829	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
8830	SET_SS_DEF();
8831	}
8832	else
8833	u64EffAddr += pVCpu->cpum.GstCtx.r13;
8834	}
8835	else
8836	{
8837	uint32_t u32Disp;
8838	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8839	u64EffAddr += (int32_t)u32Disp;
8840	}
8841	break;
8842	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8843	}
8844	break;
8845	}
8846	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8847	}
8848
8849	/* Get and add the displacement. */
8850	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8851	{
8852	case 0:
8853	break;
8854	case 1:
8855	{
8856	int8_t i8Disp;
8857	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
8858	u64EffAddr += i8Disp;
8859	break;
8860	}
8861	case 2:
8862	{
8863	uint32_t u32Disp;
8864	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
8865	u64EffAddr += (int32_t)u32Disp;
8866	break;
8867	}
8868	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
8869	}
8870
8871	}
8872
8873	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
8874	*pGCPtrEff = u64EffAddr;
8875	else
8876	{
8877	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8878	*pGCPtrEff = u64EffAddr & UINT32_MAX;
8879	}
8880	}
8881
8882	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
8883	return VINF_SUCCESS;
8884	}
8885
8886
8887	/**
8888	* Calculates the effective address of a ModR/M memory operand.
8889	*
8890	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
8891	*
8892	* @return Strict VBox status code.
8893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
8894	* @param bRm The ModRM byte.
8895	* @param cbImm The size of any immediate following the
8896	* effective address opcode bytes. Important for
8897	* RIP relative addressing.
8898	* @param pGCPtrEff Where to return the effective address.
8899	* @param offRsp RSP displacement.
8900	*/
8901	VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp) RT_NOEXCEPT
8902	{
8903	Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
8904	# define SET_SS_DEF() \
8905	do \
8906	{ \
8907	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
8908	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
8909	} while (0)
8910
8911	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
8912	{
8913	/** @todo Check the effective address size crap! */
8914	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
8915	{
8916	uint16_t u16EffAddr;
8917
8918	/* Handle the disp16 form with no registers first. */
8919	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
8920	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
8921	else
8922	{
8923	/* Get the displacment. */
8924	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
8925	{
8926	case 0: u16EffAddr = 0; break;
8927	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
8928	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
8929	default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
8930	}
8931
8932	/* Add the base and index registers to the disp. */
8933	switch (bRm & X86_MODRM_RM_MASK)
8934	{
8935	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
8936	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
8937	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
8938	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
8939	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
8940	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
8941	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
8942	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
8943	}
8944	}
8945
8946	*pGCPtrEff = u16EffAddr;
8947	}
8948	else
8949	{
8950	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
8951	uint32_t u32EffAddr;
8952
8953	/* Handle the disp32 form with no registers first. */
8954	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
8955	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
8956	else
8957	{
8958	/* Get the register (or SIB) value. */
8959	switch ((bRm & X86_MODRM_RM_MASK))
8960	{
8961	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8962	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8963	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8964	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8965	case 4: /* SIB */
8966	{
8967	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
8968
8969	/* Get the index and scale it. */
8970	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
8971	{
8972	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
8973	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
8974	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
8975	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
8976	case 4: u32EffAddr = 0; /none / break;
8977	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
8978	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
8979	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
8980	IEM_NOT_REACHED_DEFAULT_CASE_RET();
8981	}
8982	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
8983
8984	/* add base */
8985	switch (bSib & X86_SIB_BASE_MASK)
8986	{
8987	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
8988	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
8989	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
8990	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
8991	case 4:
8992	u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
8993	SET_SS_DEF();
8994	break;
8995	case 5:
8996	if ((bRm & X86_MODRM_MOD_MASK) != 0)
8997	{
8998	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
8999	SET_SS_DEF();
9000	}
9001	else
9002	{
9003	uint32_t u32Disp;
9004	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9005	u32EffAddr += u32Disp;
9006	}
9007	break;
9008	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
9009	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
9010	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9011	}
9012	break;
9013	}
9014	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
9015	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9016	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9017	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9018	}
9019
9020	/* Get and add the displacement. */
9021	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9022	{
9023	case 0:
9024	break;
9025	case 1:
9026	{
9027	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9028	u32EffAddr += i8Disp;
9029	break;
9030	}
9031	case 2:
9032	{
9033	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9034	u32EffAddr += u32Disp;
9035	break;
9036	}
9037	default:
9038	AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
9039	}
9040
9041	}
9042	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9043	*pGCPtrEff = u32EffAddr;
9044	else
9045	{
9046	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9047	*pGCPtrEff = u32EffAddr & UINT16_MAX;
9048	}
9049	}
9050	}
9051	else
9052	{
9053	uint64_t u64EffAddr;
9054
9055	/* Handle the rip+disp32 form with no registers first. */
9056	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9057	{
9058	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9059	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9060	}
9061	else
9062	{
9063	/* Get the register (or SIB) value. */
9064	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
9065	{
9066	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9067	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9068	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9069	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9070	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9071	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9072	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9073	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9074	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9075	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9076	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9077	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9078	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9079	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9080	/* SIB */
9081	case 4:
9082	case 12:
9083	{
9084	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9085
9086	/* Get the index and scale it. */
9087	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
9088	{
9089	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9090	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9091	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9092	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9093	case 4: u64EffAddr = 0; /none / break;
9094	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9095	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9096	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9097	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9098	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9099	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9100	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9101	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9102	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9103	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9104	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9105	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9106	}
9107	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9108
9109	/* add base */
9110	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
9111	{
9112	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9113	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9114	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9115	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9116	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
9117	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9118	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9119	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9120	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9121	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9122	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9123	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9124	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9125	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9126	/* complicated encodings */
9127	case 5:
9128	case 13:
9129	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9130	{
9131	if (!pVCpu->iem.s.uRexB)
9132	{
9133	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9134	SET_SS_DEF();
9135	}
9136	else
9137	u64EffAddr += pVCpu->cpum.GstCtx.r13;
9138	}
9139	else
9140	{
9141	uint32_t u32Disp;
9142	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9143	u64EffAddr += (int32_t)u32Disp;
9144	}
9145	break;
9146	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9147	}
9148	break;
9149	}
9150	IEM_NOT_REACHED_DEFAULT_CASE_RET();
9151	}
9152
9153	/* Get and add the displacement. */
9154	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9155	{
9156	case 0:
9157	break;
9158	case 1:
9159	{
9160	int8_t i8Disp;
9161	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9162	u64EffAddr += i8Disp;
9163	break;
9164	}
9165	case 2:
9166	{
9167	uint32_t u32Disp;
9168	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9169	u64EffAddr += (int32_t)u32Disp;
9170	break;
9171	}
9172	IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
9173	}
9174
9175	}
9176
9177	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9178	*pGCPtrEff = u64EffAddr;
9179	else
9180	{
9181	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9182	*pGCPtrEff = u64EffAddr & UINT32_MAX;
9183	}
9184	}
9185
9186	Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
9187	return VINF_SUCCESS;
9188	}
9189
9190
9191	#ifdef IEM_WITH_SETJMP
9192	/**
9193	* Calculates the effective address of a ModR/M memory operand.
9194	*
9195	* Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
9196	*
9197	* May longjmp on internal error.
9198	*
9199	* @return The effective address.
9200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9201	* @param bRm The ModRM byte.
9202	* @param cbImm The size of any immediate following the
9203	* effective address opcode bytes. Important for
9204	* RIP relative addressing.
9205	*/
9206	RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm) RT_NOEXCEPT
9207	{
9208	Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
9209	# define SET_SS_DEF() \
9210	do \
9211	{ \
9212	if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
9213	pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
9214	} while (0)
9215
9216	if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
9217	{
9218	/** @todo Check the effective address size crap! */
9219	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
9220	{
9221	uint16_t u16EffAddr;
9222
9223	/* Handle the disp16 form with no registers first. */
9224	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 6)
9225	IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
9226	else
9227	{
9228	/* Get the displacment. */
9229	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9230	{
9231	case 0: u16EffAddr = 0; break;
9232	case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
9233	case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
9234	default: AssertFailedStmt(longjmp(pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); / (caller checked for these) */
9235	}
9236
9237	/* Add the base and index registers to the disp. */
9238	switch (bRm & X86_MODRM_RM_MASK)
9239	{
9240	case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
9241	case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
9242	case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
9243	case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
9244	case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
9245	case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
9246	case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
9247	case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
9248	}
9249	}
9250
9251	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
9252	return u16EffAddr;
9253	}
9254
9255	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9256	uint32_t u32EffAddr;
9257
9258	/* Handle the disp32 form with no registers first. */
9259	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9260	IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
9261	else
9262	{
9263	/* Get the register (or SIB) value. */
9264	switch ((bRm & X86_MODRM_RM_MASK))
9265	{
9266	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9267	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9268	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9269	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9270	case 4: /* SIB */
9271	{
9272	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9273
9274	/* Get the index and scale it. */
9275	switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
9276	{
9277	case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
9278	case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
9279	case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
9280	case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
9281	case 4: u32EffAddr = 0; /none / break;
9282	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
9283	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9284	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9285	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9286	}
9287	u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9288
9289	/* add base */
9290	switch (bSib & X86_SIB_BASE_MASK)
9291	{
9292	case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
9293	case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
9294	case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
9295	case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
9296	case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
9297	case 5:
9298	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9299	{
9300	u32EffAddr += pVCpu->cpum.GstCtx.ebp;
9301	SET_SS_DEF();
9302	}
9303	else
9304	{
9305	uint32_t u32Disp;
9306	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9307	u32EffAddr += u32Disp;
9308	}
9309	break;
9310	case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
9311	case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
9312	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9313	}
9314	break;
9315	}
9316	case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
9317	case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
9318	case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
9319	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9320	}
9321
9322	/* Get and add the displacement. */
9323	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9324	{
9325	case 0:
9326	break;
9327	case 1:
9328	{
9329	int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9330	u32EffAddr += i8Disp;
9331	break;
9332	}
9333	case 2:
9334	{
9335	uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9336	u32EffAddr += u32Disp;
9337	break;
9338	}
9339	default:
9340	AssertFailedStmt(longjmp(pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); / (caller checked for these) */
9341	}
9342	}
9343
9344	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
9345	{
9346	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
9347	return u32EffAddr;
9348	}
9349	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
9350	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
9351	return u32EffAddr & UINT16_MAX;
9352	}
9353
9354	uint64_t u64EffAddr;
9355
9356	/* Handle the rip+disp32 form with no registers first. */
9357	if ((bRm & (X86_MODRM_MOD_MASK \| X86_MODRM_RM_MASK)) == 5)
9358	{
9359	IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
9360	u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
9361	}
9362	else
9363	{
9364	/* Get the register (or SIB) value. */
9365	switch ((bRm & X86_MODRM_RM_MASK) \| pVCpu->iem.s.uRexB)
9366	{
9367	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9368	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9369	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9370	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9371	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
9372	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9373	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9374	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9375	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9376	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9377	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9378	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9379	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9380	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9381	/* SIB */
9382	case 4:
9383	case 12:
9384	{
9385	uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
9386
9387	/* Get the index and scale it. */
9388	switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) \| pVCpu->iem.s.uRexIndex)
9389	{
9390	case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
9391	case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
9392	case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
9393	case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
9394	case 4: u64EffAddr = 0; /none / break;
9395	case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
9396	case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
9397	case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
9398	case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
9399	case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
9400	case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
9401	case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
9402	case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
9403	case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
9404	case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
9405	case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
9406	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9407	}
9408	u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
9409
9410	/* add base */
9411	switch ((bSib & X86_SIB_BASE_MASK) \| pVCpu->iem.s.uRexB)
9412	{
9413	case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
9414	case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
9415	case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
9416	case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
9417	case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
9418	case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
9419	case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
9420	case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
9421	case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
9422	case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
9423	case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
9424	case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
9425	case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
9426	case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
9427	/* complicated encodings */
9428	case 5:
9429	case 13:
9430	if ((bRm & X86_MODRM_MOD_MASK) != 0)
9431	{
9432	if (!pVCpu->iem.s.uRexB)
9433	{
9434	u64EffAddr += pVCpu->cpum.GstCtx.rbp;
9435	SET_SS_DEF();
9436	}
9437	else
9438	u64EffAddr += pVCpu->cpum.GstCtx.r13;
9439	}
9440	else
9441	{
9442	uint32_t u32Disp;
9443	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9444	u64EffAddr += (int32_t)u32Disp;
9445	}
9446	break;
9447	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9448	}
9449	break;
9450	}
9451	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
9452	}
9453
9454	/* Get and add the displacement. */
9455	switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
9456	{
9457	case 0:
9458	break;
9459	case 1:
9460	{
9461	int8_t i8Disp;
9462	IEM_OPCODE_GET_NEXT_S8(&i8Disp);
9463	u64EffAddr += i8Disp;
9464	break;
9465	}
9466	case 2:
9467	{
9468	uint32_t u32Disp;
9469	IEM_OPCODE_GET_NEXT_U32(&u32Disp);
9470	u64EffAddr += (int32_t)u32Disp;
9471	break;
9472	}
9473	IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
9474	}
9475
9476	}
9477
9478	if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
9479	{
9480	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
9481	return u64EffAddr;
9482	}
9483	Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
9484	Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
9485	return u64EffAddr & UINT32_MAX;
9486	}
9487	#endif /* IEM_WITH_SETJMP */
9488
9489	/** @} */
9490
9491
9492	#ifdef LOG_ENABLED
9493	/**
9494	* Logs the current instruction.
9495	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9496	* @param fSameCtx Set if we have the same context information as the VMM,
9497	* clear if we may have already executed an instruction in
9498	* our debug context. When clear, we assume IEMCPU holds
9499	* valid CPU mode info.
9500	*
9501	* The @a fSameCtx parameter is now misleading and obsolete.
9502	* @param pszFunction The IEM function doing the execution.
9503	*/
9504	static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT
9505	{
9506	# ifdef IN_RING3
9507	if (LogIs2Enabled())
9508	{
9509	char szInstr[256];
9510	uint32_t cbInstr = 0;
9511	if (fSameCtx)
9512	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
9513	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
9514	szInstr, sizeof(szInstr), &cbInstr);
9515	else
9516	{
9517	uint32_t fFlags = 0;
9518	switch (pVCpu->iem.s.enmCpuMode)
9519	{
9520	case IEMMODE_64BIT: fFlags \|= DBGF_DISAS_FLAGS_64BIT_MODE; break;
9521	case IEMMODE_32BIT: fFlags \|= DBGF_DISAS_FLAGS_32BIT_MODE; break;
9522	case IEMMODE_16BIT:
9523	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
9524	fFlags \|= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
9525	else
9526	fFlags \|= DBGF_DISAS_FLAGS_16BIT_MODE;
9527	break;
9528	}
9529	DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
9530	szInstr, sizeof(szInstr), &cbInstr);
9531	}
9532
9533	PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
9534	Log2(("**** %s\n"
9535	" eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
9536	" eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
9537	" cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
9538	" fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
9539	" %s\n"
9540	, pszFunction,
9541	pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
9542	pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
9543	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
9544	pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
9545	pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
9546	szInstr));
9547
9548	if (LogIs3Enabled())
9549	DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
9550	}
9551	else
9552	# endif
9553	LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
9554	pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
9555	RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
9556	}
9557	#endif /* LOG_ENABLED */
9558
9559
9560	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9561	/**
9562	* Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
9563	* VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
9564	*
9565	* @returns Modified rcStrict.
9566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
9567	* @param rcStrict The instruction execution status.
9568	*/
9569	static VBOXSTRICTRC iemHandleNestedInstructionBoundaryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
9570	{
9571	Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
9572	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF))
9573	{
9574	/* VMX preemption timer takes priority over NMI-window exits. */
9575	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
9576	{
9577	rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
9578	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
9579	}
9580	/*
9581	* Check remaining intercepts.
9582	*
9583	* NMI-window and Interrupt-window VM-exits.
9584	* Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
9585	* Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
9586	*
9587	* See Intel spec. 26.7.6 "NMI-Window Exiting".
9588	* See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
9589	*/
9590	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW \| VMCPU_FF_VMX_INT_WINDOW)
9591	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9592	&& !TRPMHasTrap(pVCpu))
9593	{
9594	Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
9595	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
9596	&& CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
9597	{
9598	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
9599	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
9600	}
9601	else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
9602	&& CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
9603	{
9604	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
9605	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
9606	}
9607	}
9608	}
9609	/* TPR-below threshold/APIC write has the highest priority. */
9610	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
9611	{
9612	rcStrict = iemVmxApicWriteEmulation(pVCpu);
9613	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9614	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
9615	}
9616	/* MTF takes priority over VMX-preemption timer. */
9617	else
9618	{
9619	rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
9620	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9621	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
9622	}
9623	return rcStrict;
9624	}
9625	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
9626
9627
9628	/**
9629	* The actual code execution bits of IEMExecOne, IEMExecOneEx, and
9630	* IEMExecOneWithPrefetchedByPC.
9631	*
9632	* Similar code is found in IEMExecLots.
9633	*
9634	* @return Strict VBox status code.
9635	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9636	* @param fExecuteInhibit If set, execute the instruction following CLI,
9637	* POP SS and MOV SS,GR.
9638	* @param pszFunction The calling function name.
9639	*/
9640	DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
9641	{
9642	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9643	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9644	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9645	RT_NOREF_PV(pszFunction);
9646
9647	#ifdef IEM_WITH_SETJMP
9648	VBOXSTRICTRC rcStrict;
9649	jmp_buf JmpBuf;
9650	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
9651	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9652	if ((rcStrict = setjmp(JmpBuf)) == 0)
9653	{
9654	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9655	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9656	}
9657	else
9658	pVCpu->iem.s.cLongJumps++;
9659	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9660	#else
9661	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9662	VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9663	#endif
9664	if (rcStrict == VINF_SUCCESS)
9665	pVCpu->iem.s.cInstructions++;
9666	if (pVCpu->iem.s.cActiveMappings > 0)
9667	{
9668	Assert(rcStrict != VINF_SUCCESS);
9669	iemMemRollback(pVCpu);
9670	}
9671	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9672	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9673	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9674
9675	//#ifdef DEBUG
9676	// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr \|\| rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
9677	//#endif
9678
9679	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9680	/*
9681	* Perform any VMX nested-guest instruction boundary actions.
9682	*
9683	* If any of these causes a VM-exit, we must skip executing the next
9684	* instruction (would run into stale page tables). A VM-exit makes sure
9685	* there is no interrupt-inhibition, so that should ensure we don't go
9686	* to try execute the next instruction. Clearing fExecuteInhibit is
9687	* problematic because of the setjmp/longjmp clobbering above.
9688	*/
9689	if ( !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9690	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW)
9691	\|\| rcStrict != VINF_SUCCESS)
9692	{ /* likely */ }
9693	else
9694	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9695	#endif
9696
9697	/* Execute the next instruction as well if a cli, pop ss or
9698	mov ss, Gr has just completed successfully. */
9699	if ( fExecuteInhibit
9700	&& rcStrict == VINF_SUCCESS
9701	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
9702	&& EMIsInhibitInterruptsActive(pVCpu))
9703	{
9704	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock);
9705	if (rcStrict == VINF_SUCCESS)
9706	{
9707	#ifdef LOG_ENABLED
9708	iemLogCurInstr(pVCpu, false, pszFunction);
9709	#endif
9710	#ifdef IEM_WITH_SETJMP
9711	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
9712	if ((rcStrict = setjmp(JmpBuf)) == 0)
9713	{
9714	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
9715	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9716	}
9717	else
9718	pVCpu->iem.s.cLongJumps++;
9719	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
9720	#else
9721	IEM_OPCODE_GET_NEXT_U8(&b);
9722	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
9723	#endif
9724	if (rcStrict == VINF_SUCCESS)
9725	{
9726	pVCpu->iem.s.cInstructions++;
9727	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9728	if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
9729	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW))
9730	{ /* likely */ }
9731	else
9732	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
9733	#endif
9734	}
9735	if (pVCpu->iem.s.cActiveMappings > 0)
9736	{
9737	Assert(rcStrict != VINF_SUCCESS);
9738	iemMemRollback(pVCpu);
9739	}
9740	AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
9741	AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
9742	AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
9743	}
9744	else if (pVCpu->iem.s.cActiveMappings > 0)
9745	iemMemRollback(pVCpu);
9746	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
9747	}
9748
9749	/*
9750	* Return value fiddling, statistics and sanity assertions.
9751	*/
9752	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
9753
9754	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
9755	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
9756	return rcStrict;
9757	}
9758
9759
9760	/**
9761	* Execute one instruction.
9762	*
9763	* @return Strict VBox status code.
9764	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9765	*/
9766	VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
9767	{
9768	AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
9769	#ifdef LOG_ENABLED
9770	iemLogCurInstr(pVCpu, true, "IEMExecOne");
9771	#endif
9772
9773	/*
9774	* Do the decoding and emulation.
9775	*/
9776	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9777	if (rcStrict == VINF_SUCCESS)
9778	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
9779	else if (pVCpu->iem.s.cActiveMappings > 0)
9780	iemMemRollback(pVCpu);
9781
9782	if (rcStrict != VINF_SUCCESS)
9783	LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9784	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9785	return rcStrict;
9786	}
9787
9788
9789	VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9790	{
9791	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9792
9793	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9794	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9795	if (rcStrict == VINF_SUCCESS)
9796	{
9797	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
9798	if (pcbWritten)
9799	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9800	}
9801	else if (pVCpu->iem.s.cActiveMappings > 0)
9802	iemMemRollback(pVCpu);
9803
9804	return rcStrict;
9805	}
9806
9807
9808	VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9809	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9810	{
9811	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9812
9813	VBOXSTRICTRC rcStrict;
9814	if ( cbOpcodeBytes
9815	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9816	{
9817	iemInitDecoder(pVCpu, false, false);
9818	#ifdef IEM_WITH_CODE_TLB
9819	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9820	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9821	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9822	pVCpu->iem.s.offCurInstrStart = 0;
9823	pVCpu->iem.s.offInstrNextByte = 0;
9824	#else
9825	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9826	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9827	#endif
9828	rcStrict = VINF_SUCCESS;
9829	}
9830	else
9831	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
9832	if (rcStrict == VINF_SUCCESS)
9833	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
9834	else if (pVCpu->iem.s.cActiveMappings > 0)
9835	iemMemRollback(pVCpu);
9836
9837	return rcStrict;
9838	}
9839
9840
9841	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
9842	{
9843	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9844
9845	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9846	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9847	if (rcStrict == VINF_SUCCESS)
9848	{
9849	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
9850	if (pcbWritten)
9851	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9852	}
9853	else if (pVCpu->iem.s.cActiveMappings > 0)
9854	iemMemRollback(pVCpu);
9855
9856	return rcStrict;
9857	}
9858
9859
9860	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9861	const void *pvOpcodeBytes, size_t cbOpcodeBytes)
9862	{
9863	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9864
9865	VBOXSTRICTRC rcStrict;
9866	if ( cbOpcodeBytes
9867	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9868	{
9869	iemInitDecoder(pVCpu, true, false);
9870	#ifdef IEM_WITH_CODE_TLB
9871	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9872	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9873	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9874	pVCpu->iem.s.offCurInstrStart = 0;
9875	pVCpu->iem.s.offInstrNextByte = 0;
9876	#else
9877	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9878	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9879	#endif
9880	rcStrict = VINF_SUCCESS;
9881	}
9882	else
9883	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9884	if (rcStrict == VINF_SUCCESS)
9885	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
9886	else if (pVCpu->iem.s.cActiveMappings > 0)
9887	iemMemRollback(pVCpu);
9888
9889	return rcStrict;
9890	}
9891
9892
9893	/**
9894	* For debugging DISGetParamSize, may come in handy.
9895	*
9896	* @returns Strict VBox status code.
9897	* @param pVCpu The cross context virtual CPU structure of the
9898	* calling EMT.
9899	* @param pCtxCore The context core structure.
9900	* @param OpcodeBytesPC The PC of the opcode bytes.
9901	* @param pvOpcodeBytes Prefeched opcode bytes.
9902	* @param cbOpcodeBytes Number of prefetched bytes.
9903	* @param pcbWritten Where to return the number of bytes written.
9904	* Optional.
9905	*/
9906	VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
9907	const void *pvOpcodeBytes, size_t cbOpcodeBytes,
9908	uint32_t *pcbWritten)
9909	{
9910	AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
9911
9912	uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
9913	VBOXSTRICTRC rcStrict;
9914	if ( cbOpcodeBytes
9915	&& pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
9916	{
9917	iemInitDecoder(pVCpu, true, false);
9918	#ifdef IEM_WITH_CODE_TLB
9919	pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
9920	pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
9921	pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
9922	pVCpu->iem.s.offCurInstrStart = 0;
9923	pVCpu->iem.s.offInstrNextByte = 0;
9924	#else
9925	pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
9926	memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
9927	#endif
9928	rcStrict = VINF_SUCCESS;
9929	}
9930	else
9931	rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
9932	if (rcStrict == VINF_SUCCESS)
9933	{
9934	rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
9935	if (pcbWritten)
9936	*pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
9937	}
9938	else if (pVCpu->iem.s.cActiveMappings > 0)
9939	iemMemRollback(pVCpu);
9940
9941	return rcStrict;
9942	}
9943
9944
9945	/**
9946	* For handling split cacheline lock operations when the host has split-lock
9947	* detection enabled.
9948	*
9949	* This will cause the interpreter to disregard the lock prefix and implicit
9950	* locking (xchg).
9951	*
9952	* @returns Strict VBox status code.
9953	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
9954	*/
9955	VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
9956	{
9957	/*
9958	* Do the decoding and emulation.
9959	*/
9960	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /fDisregardLock/);
9961	if (rcStrict == VINF_SUCCESS)
9962	rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
9963	else if (pVCpu->iem.s.cActiveMappings > 0)
9964	iemMemRollback(pVCpu);
9965
9966	if (rcStrict != VINF_SUCCESS)
9967	LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
9968	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
9969	return rcStrict;
9970	}
9971
9972
9973	VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
9974	{
9975	uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
9976	AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
9977
9978	/*
9979	* See if there is an interrupt pending in TRPM, inject it if we can.
9980	*/
9981	/** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
9982	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
9983	bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
9984	if (fIntrEnabled)
9985	{
9986	if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
9987	fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9988	else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
9989	fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
9990	else
9991	{
9992	Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
9993	fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
9994	}
9995	}
9996	#else
9997	bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
9998	#endif
9999
10000	/** @todo What if we are injecting an exception and not an interrupt? Is that
10001	* possible here? For now we assert it is indeed only an interrupt. */
10002	if ( fIntrEnabled
10003	&& TRPMHasTrap(pVCpu)
10004	&& EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
10005	{
10006	uint8_t u8TrapNo;
10007	TRPMEVENT enmType;
10008	uint32_t uErrCode;
10009	RTGCPTR uCr2;
10010	int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen /, NULL / fIcebp */);
10011	AssertRC(rc2);
10012	Assert(enmType == TRPM_HARDWARE_INT);
10013	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
10014	TRPMResetTrap(pVCpu);
10015	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10016	/* Injecting an event may cause a VM-exit. */
10017	if ( rcStrict != VINF_SUCCESS
10018	&& rcStrict != VINF_IEM_RAISED_XCPT)
10019	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
10020	#else
10021	NOREF(rcStrict);
10022	#endif
10023	}
10024
10025	/*
10026	* Initial decoder init w/ prefetch, then setup setjmp.
10027	*/
10028	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10029	if (rcStrict == VINF_SUCCESS)
10030	{
10031	#ifdef IEM_WITH_SETJMP
10032	jmp_buf JmpBuf;
10033	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10034	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10035	pVCpu->iem.s.cActiveMappings = 0;
10036	if ((rcStrict = setjmp(JmpBuf)) == 0)
10037	#endif
10038	{
10039	/*
10040	* The run loop. We limit ourselves to 4096 instructions right now.
10041	*/
10042	uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
10043	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
10044	for (;;)
10045	{
10046	/*
10047	* Log the state.
10048	*/
10049	#ifdef LOG_ENABLED
10050	iemLogCurInstr(pVCpu, true, "IEMExecLots");
10051	#endif
10052
10053	/*
10054	* Do the decoding and emulation.
10055	*/
10056	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10057	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10058	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10059	{
10060	Assert(pVCpu->iem.s.cActiveMappings == 0);
10061	pVCpu->iem.s.cInstructions++;
10062
10063	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10064	/* Perform any VMX nested-guest instruction boundary actions. */
10065	uint64_t fCpu = pVCpu->fLocalForcedActions;
10066	if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
10067	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW)))
10068	{ /* likely */ }
10069	else
10070	{
10071	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
10072	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10073	fCpu = pVCpu->fLocalForcedActions;
10074	else
10075	{
10076	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10077	break;
10078	}
10079	}
10080	#endif
10081	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10082	{
10083	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
10084	uint64_t fCpu = pVCpu->fLocalForcedActions;
10085	#endif
10086	fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10087	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10088	\| VMCPU_FF_TLB_FLUSH
10089	\| VMCPU_FF_INHIBIT_INTERRUPTS
10090	\| VMCPU_FF_BLOCK_NMIS
10091	\| VMCPU_FF_UNHALT );
10092
10093	if (RT_LIKELY( ( !fCpu
10094	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
10095	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
10096	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
10097	{
10098	if (cMaxInstructionsGccStupidity-- > 0)
10099	{
10100	/* Poll timers every now an then according to the caller's specs. */
10101	if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
10102	\|\| !TMTimerPollBool(pVM, pVCpu))
10103	{
10104	Assert(pVCpu->iem.s.cActiveMappings == 0);
10105	iemReInitDecoder(pVCpu);
10106	continue;
10107	}
10108	}
10109	}
10110	}
10111	Assert(pVCpu->iem.s.cActiveMappings == 0);
10112	}
10113	else if (pVCpu->iem.s.cActiveMappings > 0)
10114	iemMemRollback(pVCpu);
10115	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10116	break;
10117	}
10118	}
10119	#ifdef IEM_WITH_SETJMP
10120	else
10121	{
10122	if (pVCpu->iem.s.cActiveMappings > 0)
10123	iemMemRollback(pVCpu);
10124	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10125	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10126	# endif
10127	pVCpu->iem.s.cLongJumps++;
10128	}
10129	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10130	#endif
10131
10132	/*
10133	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10134	*/
10135	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10136	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10137	}
10138	else
10139	{
10140	if (pVCpu->iem.s.cActiveMappings > 0)
10141	iemMemRollback(pVCpu);
10142
10143	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10144	/*
10145	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10146	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10147	*/
10148	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10149	#endif
10150	}
10151
10152	/*
10153	* Maybe re-enter raw-mode and log.
10154	*/
10155	if (rcStrict != VINF_SUCCESS)
10156	LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10157	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10158	if (pcInstructions)
10159	*pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
10160	return rcStrict;
10161	}
10162
10163
10164	/**
10165	* Interface used by EMExecuteExec, does exit statistics and limits.
10166	*
10167	* @returns Strict VBox status code.
10168	* @param pVCpu The cross context virtual CPU structure.
10169	* @param fWillExit To be defined.
10170	* @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
10171	* @param cMaxInstructions Maximum number of instructions to execute.
10172	* @param cMaxInstructionsWithoutExits
10173	* The max number of instructions without exits.
10174	* @param pStats Where to return statistics.
10175	*/
10176	VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
10177	uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
10178	{
10179	NOREF(fWillExit); /** @todo define flexible exit crits */
10180
10181	/*
10182	* Initialize return stats.
10183	*/
10184	pStats->cInstructions = 0;
10185	pStats->cExits = 0;
10186	pStats->cMaxExitDistance = 0;
10187	pStats->cReserved = 0;
10188
10189	/*
10190	* Initial decoder init w/ prefetch, then setup setjmp.
10191	*/
10192	VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
10193	if (rcStrict == VINF_SUCCESS)
10194	{
10195	#ifdef IEM_WITH_SETJMP
10196	jmp_buf JmpBuf;
10197	jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
10198	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
10199	pVCpu->iem.s.cActiveMappings = 0;
10200	if ((rcStrict = setjmp(JmpBuf)) == 0)
10201	#endif
10202	{
10203	#ifdef IN_RING0
10204	bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() \|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
10205	#endif
10206	uint32_t cInstructionSinceLastExit = 0;
10207
10208	/*
10209	* The run loop. We limit ourselves to 4096 instructions right now.
10210	*/
10211	PVM pVM = pVCpu->CTX_SUFF(pVM);
10212	for (;;)
10213	{
10214	/*
10215	* Log the state.
10216	*/
10217	#ifdef LOG_ENABLED
10218	iemLogCurInstr(pVCpu, true, "IEMExecForExits");
10219	#endif
10220
10221	/*
10222	* Do the decoding and emulation.
10223	*/
10224	uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
10225
10226	uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
10227	rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
10228
10229	if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
10230	&& cInstructionSinceLastExit > 0 /* don't count the first */ )
10231	{
10232	pStats->cExits += 1;
10233	if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
10234	pStats->cMaxExitDistance = cInstructionSinceLastExit;
10235	cInstructionSinceLastExit = 0;
10236	}
10237
10238	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10239	{
10240	Assert(pVCpu->iem.s.cActiveMappings == 0);
10241	pVCpu->iem.s.cInstructions++;
10242	pStats->cInstructions++;
10243	cInstructionSinceLastExit++;
10244
10245	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10246	/* Perform any VMX nested-guest instruction boundary actions. */
10247	uint64_t fCpu = pVCpu->fLocalForcedActions;
10248	if (!(fCpu & ( VMCPU_FF_VMX_APIC_WRITE \| VMCPU_FF_VMX_MTF \| VMCPU_FF_VMX_PREEMPT_TIMER
10249	\| VMCPU_FF_VMX_INT_WINDOW \| VMCPU_FF_VMX_NMI_WINDOW)))
10250	{ /* likely */ }
10251	else
10252	{
10253	rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
10254	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10255	fCpu = pVCpu->fLocalForcedActions;
10256	else
10257	{
10258	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10259	break;
10260	}
10261	}
10262	#endif
10263	if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
10264	{
10265	#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
10266	uint64_t fCpu = pVCpu->fLocalForcedActions;
10267	#endif
10268	fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
10269	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
10270	\| VMCPU_FF_TLB_FLUSH
10271	\| VMCPU_FF_INHIBIT_INTERRUPTS
10272	\| VMCPU_FF_BLOCK_NMIS
10273	\| VMCPU_FF_UNHALT );
10274	if (RT_LIKELY( ( ( !fCpu
10275	\|\| ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
10276	&& !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
10277	&& !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
10278	\|\| pStats->cInstructions < cMinInstructions))
10279	{
10280	if (pStats->cInstructions < cMaxInstructions)
10281	{
10282	if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
10283	{
10284	#ifdef IN_RING0
10285	if ( !fCheckPreemptionPending
10286	\|\| !RTThreadPreemptIsPending(NIL_RTTHREAD))
10287	#endif
10288	{
10289	Assert(pVCpu->iem.s.cActiveMappings == 0);
10290	iemReInitDecoder(pVCpu);
10291	continue;
10292	}
10293	#ifdef IN_RING0
10294	rcStrict = VINF_EM_RAW_INTERRUPT;
10295	break;
10296	#endif
10297	}
10298	}
10299	}
10300	Assert(!(fCpu & VMCPU_FF_IEM));
10301	}
10302	Assert(pVCpu->iem.s.cActiveMappings == 0);
10303	}
10304	else if (pVCpu->iem.s.cActiveMappings > 0)
10305	iemMemRollback(pVCpu);
10306	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10307	break;
10308	}
10309	}
10310	#ifdef IEM_WITH_SETJMP
10311	else
10312	{
10313	if (pVCpu->iem.s.cActiveMappings > 0)
10314	iemMemRollback(pVCpu);
10315	pVCpu->iem.s.cLongJumps++;
10316	}
10317	pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
10318	#endif
10319
10320	/*
10321	* Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
10322	*/
10323	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
10324	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
10325	}
10326	else
10327	{
10328	if (pVCpu->iem.s.cActiveMappings > 0)
10329	iemMemRollback(pVCpu);
10330
10331	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
10332	/*
10333	* When a nested-guest causes an exception intercept (e.g. #PF) when fetching
10334	* code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
10335	*/
10336	rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
10337	#endif
10338	}
10339
10340	/*
10341	* Maybe re-enter raw-mode and log.
10342	*/
10343	if (rcStrict != VINF_SUCCESS)
10344	LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
10345	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
10346	pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
10347	return rcStrict;
10348	}
10349
10350
10351	/**
10352	* Injects a trap, fault, abort, software interrupt or external interrupt.
10353	*
10354	* The parameter list matches TRPMQueryTrapAll pretty closely.
10355	*
10356	* @returns Strict VBox status code.
10357	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10358	* @param u8TrapNo The trap number.
10359	* @param enmType What type is it (trap/fault/abort), software
10360	* interrupt or hardware interrupt.
10361	* @param uErrCode The error code if applicable.
10362	* @param uCr2 The CR2 value if applicable.
10363	* @param cbInstr The instruction length (only relevant for
10364	* software interrupts).
10365	*/
10366	VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
10367	uint8_t cbInstr)
10368	{
10369	iemInitDecoder(pVCpu, false, false);
10370	#ifdef DBGFTRACE_ENABLED
10371	RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
10372	u8TrapNo, enmType, uErrCode, uCr2);
10373	#endif
10374
10375	uint32_t fFlags;
10376	switch (enmType)
10377	{
10378	case TRPM_HARDWARE_INT:
10379	Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
10380	fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
10381	uErrCode = uCr2 = 0;
10382	break;
10383
10384	case TRPM_SOFTWARE_INT:
10385	Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
10386	fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
10387	uErrCode = uCr2 = 0;
10388	break;
10389
10390	case TRPM_TRAP:
10391	Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
10392	fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
10393	if (u8TrapNo == X86_XCPT_PF)
10394	fFlags \|= IEM_XCPT_FLAGS_CR2;
10395	switch (u8TrapNo)
10396	{
10397	case X86_XCPT_DF:
10398	case X86_XCPT_TS:
10399	case X86_XCPT_NP:
10400	case X86_XCPT_SS:
10401	case X86_XCPT_PF:
10402	case X86_XCPT_AC:
10403	case X86_XCPT_GP:
10404	fFlags \|= IEM_XCPT_FLAGS_ERR;
10405	break;
10406	}
10407	break;
10408
10409	IEM_NOT_REACHED_DEFAULT_CASE_RET();
10410	}
10411
10412	VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
10413
10414	if (pVCpu->iem.s.cActiveMappings > 0)
10415	iemMemRollback(pVCpu);
10416
10417	return rcStrict;
10418	}
10419
10420
10421	/**
10422	* Injects the active TRPM event.
10423	*
10424	* @returns Strict VBox status code.
10425	* @param pVCpu The cross context virtual CPU structure.
10426	*/
10427	VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
10428	{
10429	#ifndef IEM_IMPLEMENTS_TASKSWITCH
10430	IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
10431	#else
10432	uint8_t u8TrapNo;
10433	TRPMEVENT enmType;
10434	uint32_t uErrCode;
10435	RTGCUINTPTR uCr2;
10436	uint8_t cbInstr;
10437	int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
10438	if (RT_FAILURE(rc))
10439	return rc;
10440
10441	/** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
10442	* ICEBP \#DB injection as a special case. */
10443	VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
10444	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
10445	if (rcStrict == VINF_SVM_VMEXIT)
10446	rcStrict = VINF_SUCCESS;
10447	#endif
10448	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10449	if (rcStrict == VINF_VMX_VMEXIT)
10450	rcStrict = VINF_SUCCESS;
10451	#endif
10452	/** @todo Are there any other codes that imply the event was successfully
10453	* delivered to the guest? See @bugref{6607}. */
10454	if ( rcStrict == VINF_SUCCESS
10455	\|\| rcStrict == VINF_IEM_RAISED_XCPT)
10456	TRPMResetTrap(pVCpu);
10457
10458	return rcStrict;
10459	#endif
10460	}
10461
10462
10463	VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
10464	{
10465	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10466	return VERR_NOT_IMPLEMENTED;
10467	}
10468
10469
10470	VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
10471	{
10472	RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
10473	return VERR_NOT_IMPLEMENTED;
10474	}
10475
10476
10477	#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
10478	/**
10479	* Executes a IRET instruction with default operand size.
10480	*
10481	* This is for PATM.
10482	*
10483	* @returns VBox status code.
10484	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10485	* @param pCtxCore The register frame.
10486	*/
10487	VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
10488	{
10489	PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
10490
10491	iemCtxCoreToCtx(pCtx, pCtxCore);
10492	iemInitDecoder(pVCpu);
10493	VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
10494	if (rcStrict == VINF_SUCCESS)
10495	iemCtxToCtxCore(pCtxCore, pCtx);
10496	else
10497	LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
10498	pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
10499	return rcStrict;
10500	}
10501	#endif
10502
10503
10504	/**
10505	* Interface for HM and EM for executing string I/O OUT (write) instructions.
10506	*
10507	* This API ASSUMES that the caller has already verified that the guest code is
10508	* allowed to access the I/O port. (The I/O port is in the DX register in the
10509	* guest state.)
10510	*
10511	* @returns Strict VBox status code.
10512	* @param pVCpu The cross context virtual CPU structure.
10513	* @param cbValue The size of the I/O port access (1, 2, or 4).
10514	* @param enmAddrMode The addressing mode.
10515	* @param fRepPrefix Indicates whether a repeat prefix is used
10516	* (doesn't matter which for this instruction).
10517	* @param cbInstr The instruction length in bytes.
10518	* @param iEffSeg The effective segment address.
10519	* @param fIoChecked Whether the access to the I/O port has been
10520	* checked or not. It's typically checked in the
10521	* HM scenario.
10522	*/
10523	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10524	bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
10525	{
10526	AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
10527	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10528
10529	/*
10530	* State init.
10531	*/
10532	iemInitExec(pVCpu, false /fBypassHandlers/);
10533
10534	/*
10535	* Switch orgy for getting to the right handler.
10536	*/
10537	VBOXSTRICTRC rcStrict;
10538	if (fRepPrefix)
10539	{
10540	switch (enmAddrMode)
10541	{
10542	case IEMMODE_16BIT:
10543	switch (cbValue)
10544	{
10545	case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10546	case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10547	case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10548	default:
10549	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10550	}
10551	break;
10552
10553	case IEMMODE_32BIT:
10554	switch (cbValue)
10555	{
10556	case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10557	case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10558	case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10559	default:
10560	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10561	}
10562	break;
10563
10564	case IEMMODE_64BIT:
10565	switch (cbValue)
10566	{
10567	case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10568	case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10569	case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10570	default:
10571	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10572	}
10573	break;
10574
10575	default:
10576	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10577	}
10578	}
10579	else
10580	{
10581	switch (enmAddrMode)
10582	{
10583	case IEMMODE_16BIT:
10584	switch (cbValue)
10585	{
10586	case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10587	case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10588	case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10589	default:
10590	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10591	}
10592	break;
10593
10594	case IEMMODE_32BIT:
10595	switch (cbValue)
10596	{
10597	case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10598	case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10599	case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10600	default:
10601	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10602	}
10603	break;
10604
10605	case IEMMODE_64BIT:
10606	switch (cbValue)
10607	{
10608	case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10609	case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10610	case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
10611	default:
10612	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10613	}
10614	break;
10615
10616	default:
10617	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10618	}
10619	}
10620
10621	if (pVCpu->iem.s.cActiveMappings)
10622	iemMemRollback(pVCpu);
10623
10624	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10625	}
10626
10627
10628	/**
10629	* Interface for HM and EM for executing string I/O IN (read) instructions.
10630	*
10631	* This API ASSUMES that the caller has already verified that the guest code is
10632	* allowed to access the I/O port. (The I/O port is in the DX register in the
10633	* guest state.)
10634	*
10635	* @returns Strict VBox status code.
10636	* @param pVCpu The cross context virtual CPU structure.
10637	* @param cbValue The size of the I/O port access (1, 2, or 4).
10638	* @param enmAddrMode The addressing mode.
10639	* @param fRepPrefix Indicates whether a repeat prefix is used
10640	* (doesn't matter which for this instruction).
10641	* @param cbInstr The instruction length in bytes.
10642	* @param fIoChecked Whether the access to the I/O port has been
10643	* checked or not. It's typically checked in the
10644	* HM scenario.
10645	*/
10646	VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
10647	bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
10648	{
10649	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10650
10651	/*
10652	* State init.
10653	*/
10654	iemInitExec(pVCpu, false /fBypassHandlers/);
10655
10656	/*
10657	* Switch orgy for getting to the right handler.
10658	*/
10659	VBOXSTRICTRC rcStrict;
10660	if (fRepPrefix)
10661	{
10662	switch (enmAddrMode)
10663	{
10664	case IEMMODE_16BIT:
10665	switch (cbValue)
10666	{
10667	case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10668	case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10669	case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10670	default:
10671	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10672	}
10673	break;
10674
10675	case IEMMODE_32BIT:
10676	switch (cbValue)
10677	{
10678	case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10679	case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10680	case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10681	default:
10682	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10683	}
10684	break;
10685
10686	case IEMMODE_64BIT:
10687	switch (cbValue)
10688	{
10689	case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10690	case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10691	case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10692	default:
10693	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10694	}
10695	break;
10696
10697	default:
10698	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10699	}
10700	}
10701	else
10702	{
10703	switch (enmAddrMode)
10704	{
10705	case IEMMODE_16BIT:
10706	switch (cbValue)
10707	{
10708	case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
10709	case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
10710	case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
10711	default:
10712	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10713	}
10714	break;
10715
10716	case IEMMODE_32BIT:
10717	switch (cbValue)
10718	{
10719	case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
10720	case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
10721	case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
10722	default:
10723	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10724	}
10725	break;
10726
10727	case IEMMODE_64BIT:
10728	switch (cbValue)
10729	{
10730	case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
10731	case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
10732	case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
10733	default:
10734	AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
10735	}
10736	break;
10737
10738	default:
10739	AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
10740	}
10741	}
10742
10743	if ( pVCpu->iem.s.cActiveMappings == 0
10744	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
10745	{ /* likely */ }
10746	else
10747	{
10748	AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
10749	iemMemRollback(pVCpu);
10750	}
10751	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10752	}
10753
10754
10755	/**
10756	* Interface for rawmode to write execute an OUT instruction.
10757	*
10758	* @returns Strict VBox status code.
10759	* @param pVCpu The cross context virtual CPU structure.
10760	* @param cbInstr The instruction length in bytes.
10761	* @param u16Port The port to read.
10762	* @param fImm Whether the port is specified using an immediate operand or
10763	* using the implicit DX register.
10764	* @param cbReg The register size.
10765	*
10766	* @remarks In ring-0 not all of the state needs to be synced in.
10767	*/
10768	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10769	{
10770	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10771	Assert(cbReg <= 4 && cbReg != 3);
10772
10773	iemInitExec(pVCpu, false /fBypassHandlers/);
10774	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
10775	Assert(!pVCpu->iem.s.cActiveMappings);
10776	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10777	}
10778
10779
10780	/**
10781	* Interface for rawmode to write execute an IN instruction.
10782	*
10783	* @returns Strict VBox status code.
10784	* @param pVCpu The cross context virtual CPU structure.
10785	* @param cbInstr The instruction length in bytes.
10786	* @param u16Port The port to read.
10787	* @param fImm Whether the port is specified using an immediate operand or
10788	* using the implicit DX.
10789	* @param cbReg The register size.
10790	*/
10791	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
10792	{
10793	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
10794	Assert(cbReg <= 4 && cbReg != 3);
10795
10796	iemInitExec(pVCpu, false /fBypassHandlers/);
10797	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
10798	Assert(!pVCpu->iem.s.cActiveMappings);
10799	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10800	}
10801
10802
10803	/**
10804	* Interface for HM and EM to write to a CRx register.
10805	*
10806	* @returns Strict VBox status code.
10807	* @param pVCpu The cross context virtual CPU structure.
10808	* @param cbInstr The instruction length in bytes.
10809	* @param iCrReg The control register number (destination).
10810	* @param iGReg The general purpose register number (source).
10811	*
10812	* @remarks In ring-0 not all of the state needs to be synced in.
10813	*/
10814	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
10815	{
10816	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10817	Assert(iCrReg < 16);
10818	Assert(iGReg < 16);
10819
10820	iemInitExec(pVCpu, false /fBypassHandlers/);
10821	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
10822	Assert(!pVCpu->iem.s.cActiveMappings);
10823	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10824	}
10825
10826
10827	/**
10828	* Interface for HM and EM to read from a CRx register.
10829	*
10830	* @returns Strict VBox status code.
10831	* @param pVCpu The cross context virtual CPU structure.
10832	* @param cbInstr The instruction length in bytes.
10833	* @param iGReg The general purpose register number (destination).
10834	* @param iCrReg The control register number (source).
10835	*
10836	* @remarks In ring-0 not all of the state needs to be synced in.
10837	*/
10838	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
10839	{
10840	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10841	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
10842	\| CPUMCTX_EXTRN_APIC_TPR);
10843	Assert(iCrReg < 16);
10844	Assert(iGReg < 16);
10845
10846	iemInitExec(pVCpu, false /fBypassHandlers/);
10847	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
10848	Assert(!pVCpu->iem.s.cActiveMappings);
10849	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10850	}
10851
10852
10853	/**
10854	* Interface for HM and EM to clear the CR0[TS] bit.
10855	*
10856	* @returns Strict VBox status code.
10857	* @param pVCpu The cross context virtual CPU structure.
10858	* @param cbInstr The instruction length in bytes.
10859	*
10860	* @remarks In ring-0 not all of the state needs to be synced in.
10861	*/
10862	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
10863	{
10864	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10865
10866	iemInitExec(pVCpu, false /fBypassHandlers/);
10867	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
10868	Assert(!pVCpu->iem.s.cActiveMappings);
10869	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10870	}
10871
10872
10873	/**
10874	* Interface for HM and EM to emulate the LMSW instruction (loads CR0).
10875	*
10876	* @returns Strict VBox status code.
10877	* @param pVCpu The cross context virtual CPU structure.
10878	* @param cbInstr The instruction length in bytes.
10879	* @param uValue The value to load into CR0.
10880	* @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
10881	* memory operand. Otherwise pass NIL_RTGCPTR.
10882	*
10883	* @remarks In ring-0 not all of the state needs to be synced in.
10884	*/
10885	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
10886	{
10887	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10888
10889	iemInitExec(pVCpu, false /fBypassHandlers/);
10890	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
10891	Assert(!pVCpu->iem.s.cActiveMappings);
10892	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10893	}
10894
10895
10896	/**
10897	* Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
10898	*
10899	* Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
10900	*
10901	* @returns Strict VBox status code.
10902	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10903	* @param cbInstr The instruction length in bytes.
10904	* @remarks In ring-0 not all of the state needs to be synced in.
10905	* @thread EMT(pVCpu)
10906	*/
10907	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
10908	{
10909	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10910
10911	iemInitExec(pVCpu, false /fBypassHandlers/);
10912	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
10913	Assert(!pVCpu->iem.s.cActiveMappings);
10914	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10915	}
10916
10917
10918	/**
10919	* Interface for HM and EM to emulate the WBINVD instruction.
10920	*
10921	* @returns Strict VBox status code.
10922	* @param pVCpu The cross context virtual CPU structure.
10923	* @param cbInstr The instruction length in bytes.
10924	*
10925	* @remarks In ring-0 not all of the state needs to be synced in.
10926	*/
10927	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10928	{
10929	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10930
10931	iemInitExec(pVCpu, false /fBypassHandlers/);
10932	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
10933	Assert(!pVCpu->iem.s.cActiveMappings);
10934	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10935	}
10936
10937
10938	/**
10939	* Interface for HM and EM to emulate the INVD instruction.
10940	*
10941	* @returns Strict VBox status code.
10942	* @param pVCpu The cross context virtual CPU structure.
10943	* @param cbInstr The instruction length in bytes.
10944	*
10945	* @remarks In ring-0 not all of the state needs to be synced in.
10946	*/
10947	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
10948	{
10949	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
10950
10951	iemInitExec(pVCpu, false /fBypassHandlers/);
10952	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
10953	Assert(!pVCpu->iem.s.cActiveMappings);
10954	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10955	}
10956
10957
10958	/**
10959	* Interface for HM and EM to emulate the INVLPG instruction.
10960	*
10961	* @returns Strict VBox status code.
10962	* @retval VINF_PGM_SYNC_CR3
10963	*
10964	* @param pVCpu The cross context virtual CPU structure.
10965	* @param cbInstr The instruction length in bytes.
10966	* @param GCPtrPage The effective address of the page to invalidate.
10967	*
10968	* @remarks In ring-0 not all of the state needs to be synced in.
10969	*/
10970	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
10971	{
10972	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
10973
10974	iemInitExec(pVCpu, false /fBypassHandlers/);
10975	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
10976	Assert(!pVCpu->iem.s.cActiveMappings);
10977	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
10978	}
10979
10980
10981	/**
10982	* Interface for HM and EM to emulate the INVPCID instruction.
10983	*
10984	* @returns Strict VBox status code.
10985	* @retval VINF_PGM_SYNC_CR3
10986	*
10987	* @param pVCpu The cross context virtual CPU structure.
10988	* @param cbInstr The instruction length in bytes.
10989	* @param iEffSeg The effective segment register.
10990	* @param GCPtrDesc The effective address of the INVPCID descriptor.
10991	* @param uType The invalidation type.
10992	*
10993	* @remarks In ring-0 not all of the state needs to be synced in.
10994	*/
10995	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
10996	uint64_t uType)
10997	{
10998	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
10999
11000	iemInitExec(pVCpu, false /fBypassHandlers/);
11001	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
11002	Assert(!pVCpu->iem.s.cActiveMappings);
11003	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11004	}
11005
11006
11007	/**
11008	* Interface for HM and EM to emulate the CPUID instruction.
11009	*
11010	* @returns Strict VBox status code.
11011	*
11012	* @param pVCpu The cross context virtual CPU structure.
11013	* @param cbInstr The instruction length in bytes.
11014	*
11015	* @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
11016	*/
11017	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
11018	{
11019	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11020	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
11021
11022	iemInitExec(pVCpu, false /fBypassHandlers/);
11023	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
11024	Assert(!pVCpu->iem.s.cActiveMappings);
11025	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11026	}
11027
11028
11029	/**
11030	* Interface for HM and EM to emulate the RDPMC instruction.
11031	*
11032	* @returns Strict VBox status code.
11033	*
11034	* @param pVCpu The cross context virtual CPU structure.
11035	* @param cbInstr The instruction length in bytes.
11036	*
11037	* @remarks Not all of the state needs to be synced in.
11038	*/
11039	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
11040	{
11041	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11042	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
11043
11044	iemInitExec(pVCpu, false /fBypassHandlers/);
11045	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
11046	Assert(!pVCpu->iem.s.cActiveMappings);
11047	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11048	}
11049
11050
11051	/**
11052	* Interface for HM and EM to emulate the RDTSC instruction.
11053	*
11054	* @returns Strict VBox status code.
11055	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11056	*
11057	* @param pVCpu The cross context virtual CPU structure.
11058	* @param cbInstr The instruction length in bytes.
11059	*
11060	* @remarks Not all of the state needs to be synced in.
11061	*/
11062	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
11063	{
11064	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11065	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
11066
11067	iemInitExec(pVCpu, false /fBypassHandlers/);
11068	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
11069	Assert(!pVCpu->iem.s.cActiveMappings);
11070	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11071	}
11072
11073
11074	/**
11075	* Interface for HM and EM to emulate the RDTSCP instruction.
11076	*
11077	* @returns Strict VBox status code.
11078	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11079	*
11080	* @param pVCpu The cross context virtual CPU structure.
11081	* @param cbInstr The instruction length in bytes.
11082	*
11083	* @remarks Not all of the state needs to be synced in. Recommended
11084	* to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
11085	*/
11086	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
11087	{
11088	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11089	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
11090
11091	iemInitExec(pVCpu, false /fBypassHandlers/);
11092	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
11093	Assert(!pVCpu->iem.s.cActiveMappings);
11094	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11095	}
11096
11097
11098	/**
11099	* Interface for HM and EM to emulate the RDMSR instruction.
11100	*
11101	* @returns Strict VBox status code.
11102	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11103	*
11104	* @param pVCpu The cross context virtual CPU structure.
11105	* @param cbInstr The instruction length in bytes.
11106	*
11107	* @remarks Not all of the state needs to be synced in. Requires RCX and
11108	* (currently) all MSRs.
11109	*/
11110	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11111	{
11112	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11113	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_ALL_MSRS);
11114
11115	iemInitExec(pVCpu, false /fBypassHandlers/);
11116	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
11117	Assert(!pVCpu->iem.s.cActiveMappings);
11118	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11119	}
11120
11121
11122	/**
11123	* Interface for HM and EM to emulate the WRMSR instruction.
11124	*
11125	* @returns Strict VBox status code.
11126	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11127	*
11128	* @param pVCpu The cross context virtual CPU structure.
11129	* @param cbInstr The instruction length in bytes.
11130	*
11131	* @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
11132	* and (currently) all MSRs.
11133	*/
11134	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
11135	{
11136	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
11137	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
11138	\| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_ALL_MSRS);
11139
11140	iemInitExec(pVCpu, false /fBypassHandlers/);
11141	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
11142	Assert(!pVCpu->iem.s.cActiveMappings);
11143	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11144	}
11145
11146
11147	/**
11148	* Interface for HM and EM to emulate the MONITOR instruction.
11149	*
11150	* @returns Strict VBox status code.
11151	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11152	*
11153	* @param pVCpu The cross context virtual CPU structure.
11154	* @param cbInstr The instruction length in bytes.
11155	*
11156	* @remarks Not all of the state needs to be synced in.
11157	* @remarks ASSUMES the default segment of DS and no segment override prefixes
11158	* are used.
11159	*/
11160	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
11161	{
11162	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11163	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
11164
11165	iemInitExec(pVCpu, false /fBypassHandlers/);
11166	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
11167	Assert(!pVCpu->iem.s.cActiveMappings);
11168	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11169	}
11170
11171
11172	/**
11173	* Interface for HM and EM to emulate the MWAIT instruction.
11174	*
11175	* @returns Strict VBox status code.
11176	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11177	*
11178	* @param pVCpu The cross context virtual CPU structure.
11179	* @param cbInstr The instruction length in bytes.
11180	*
11181	* @remarks Not all of the state needs to be synced in.
11182	*/
11183	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
11184	{
11185	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
11186	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RAX);
11187
11188	iemInitExec(pVCpu, false /fBypassHandlers/);
11189	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
11190	Assert(!pVCpu->iem.s.cActiveMappings);
11191	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11192	}
11193
11194
11195	/**
11196	* Interface for HM and EM to emulate the HLT instruction.
11197	*
11198	* @returns Strict VBox status code.
11199	* @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
11200	*
11201	* @param pVCpu The cross context virtual CPU structure.
11202	* @param cbInstr The instruction length in bytes.
11203	*
11204	* @remarks Not all of the state needs to be synced in.
11205	*/
11206	VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
11207	{
11208	IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
11209
11210	iemInitExec(pVCpu, false /fBypassHandlers/);
11211	VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
11212	Assert(!pVCpu->iem.s.cActiveMappings);
11213	return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
11214	}
11215
11216
11217	/**
11218	* Checks if IEM is in the process of delivering an event (interrupt or
11219	* exception).
11220	*
11221	* @returns true if we're in the process of raising an interrupt or exception,
11222	* false otherwise.
11223	* @param pVCpu The cross context virtual CPU structure.
11224	* @param puVector Where to store the vector associated with the
11225	* currently delivered event, optional.
11226	* @param pfFlags Where to store th event delivery flags (see
11227	* IEM_XCPT_FLAGS_XXX), optional.
11228	* @param puErr Where to store the error code associated with the
11229	* event, optional.
11230	* @param puCr2 Where to store the CR2 associated with the event,
11231	* optional.
11232	* @remarks The caller should check the flags to determine if the error code and
11233	* CR2 are valid for the event.
11234	*/
11235	VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t puVector, uint32_t pfFlags, uint32_t puErr, uint64_t puCr2)
11236	{
11237	bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
11238	if (fRaisingXcpt)
11239	{
11240	if (puVector)
11241	*puVector = pVCpu->iem.s.uCurXcpt;
11242	if (pfFlags)
11243	*pfFlags = pVCpu->iem.s.fCurXcpt;
11244	if (puErr)
11245	*puErr = pVCpu->iem.s.uCurXcptErr;
11246	if (puCr2)
11247	*puCr2 = pVCpu->iem.s.uCurXcptCr2;
11248	}
11249	return fRaisingXcpt;
11250	}
11251
11252	#ifdef IN_RING3
11253
11254	/**
11255	* Handles the unlikely and probably fatal merge cases.
11256	*
11257	* @returns Merged status code.
11258	* @param rcStrict Current EM status code.
11259	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11260	* with @a rcStrict.
11261	* @param iMemMap The memory mapping index. For error reporting only.
11262	* @param pVCpu The cross context virtual CPU structure of the calling
11263	* thread, for error reporting only.
11264	*/
11265	DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
11266	unsigned iMemMap, PVMCPUCC pVCpu)
11267	{
11268	if (RT_FAILURE_NP(rcStrict))
11269	return rcStrict;
11270
11271	if (RT_FAILURE_NP(rcStrictCommit))
11272	return rcStrictCommit;
11273
11274	if (rcStrict == rcStrictCommit)
11275	return rcStrictCommit;
11276
11277	AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
11278	VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
11279	pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
11280	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
11281	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
11282	return VERR_IOM_FF_STATUS_IPE;
11283	}
11284
11285
11286	/**
11287	* Helper for IOMR3ProcessForceFlag.
11288	*
11289	* @returns Merged status code.
11290	* @param rcStrict Current EM status code.
11291	* @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
11292	* with @a rcStrict.
11293	* @param iMemMap The memory mapping index. For error reporting only.
11294	* @param pVCpu The cross context virtual CPU structure of the calling
11295	* thread, for error reporting only.
11296	*/
11297	DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
11298	{
11299	/* Simple. */
11300	if (RT_LIKELY(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RAW_TO_R3))
11301	return rcStrictCommit;
11302
11303	if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
11304	return rcStrict;
11305
11306	/* EM scheduling status codes. */
11307	if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
11308	&& rcStrict <= VINF_EM_LAST))
11309	{
11310	if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
11311	&& rcStrictCommit <= VINF_EM_LAST))
11312	return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
11313	}
11314
11315	/* Unlikely */
11316	return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
11317	}
11318
11319
11320	/**
11321	* Called by force-flag handling code when VMCPU_FF_IEM is set.
11322	*
11323	* @returns Merge between @a rcStrict and what the commit operation returned.
11324	* @param pVM The cross context VM structure.
11325	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11326	* @param rcStrict The status code returned by ring-0 or raw-mode.
11327	*/
11328	VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
11329	{
11330	/*
11331	* Reset the pending commit.
11332	*/
11333	AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess \| pVCpu->iem.s.aMemMappings[1].fAccess \| pVCpu->iem.s.aMemMappings[2].fAccess)
11334	& (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND),
11335	("%#x %#x %#x\n",
11336	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11337	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
11338
11339	/*
11340	* Commit the pending bounce buffers (usually just one).
11341	*/
11342	unsigned cBufs = 0;
11343	unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
11344	while (iMemMap-- > 0)
11345	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST \| IEM_ACCESS_PENDING_R3_WRITE_2ND))
11346	{
11347	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
11348	Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
11349	Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
11350
11351	uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
11352	uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
11353	uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
11354
11355	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
11356	{
11357	VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
11358	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
11359	pbBuf,
11360	cbFirst,
11361	PGMACCESSORIGIN_IEM);
11362	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
11363	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
11364	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
11365	VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
11366	}
11367
11368	if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
11369	{
11370	VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
11371	pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
11372	pbBuf + cbFirst,
11373	cbSecond,
11374	PGMACCESSORIGIN_IEM);
11375	rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
11376	Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
11377	iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
11378	VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
11379	}
11380	cBufs++;
11381	pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
11382	}
11383
11384	AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
11385	("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
11386	pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
11387	pVCpu->iem.s.cActiveMappings = 0;
11388	return rcStrict;
11389	}
11390
11391	#endif /* IN_RING3 */
11392

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 96789

Download in other formats: