VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 93582

Last change on this file since 93582 was 93554, checked in by vboxsync, 3 years ago

VMM: Changed PAGE_SIZE -> GUEST_PAGE_SIZE / HOST_PAGE_SIZE, PAGE_SHIFT -> GUEST_PAGE_SHIFT / HOST_PAGE_SHIFT, and PAGE_OFFSET_MASK -> GUEST_PAGE_OFFSET_MASK / HOST_PAGE_OFFSET_MASK. Also removed most usage of ASMMemIsZeroPage and ASMMemZeroPage since the host and guest page size doesn't need to be the same any more. Some work left to do in the page pool code. bugref:9898
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 657.0 KB
Line 
1/* $Id: IEMAll.cpp 93554 2022-02-02 22:57:02Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - All Contexts.
4 */
5
6/*
7 * Copyright (C) 2011-2022 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/** @page pg_iem IEM - Interpreted Execution Manager
20 *
21 * The interpreted exeuction manager (IEM) is for executing short guest code
22 * sequences that are causing too many exits / virtualization traps. It will
23 * also be used to interpret single instructions, thus replacing the selective
24 * interpreters in EM and IOM.
25 *
26 * Design goals:
27 * - Relatively small footprint, although we favour speed and correctness
28 * over size.
29 * - Reasonably fast.
30 * - Correctly handle lock prefixed instructions.
31 * - Complete instruction set - eventually.
32 * - Refactorable into a recompiler, maybe.
33 * - Replace EMInterpret*.
34 *
35 * Using the existing disassembler has been considered, however this is thought
36 * to conflict with speed as the disassembler chews things a bit too much while
37 * leaving us with a somewhat complicated state to interpret afterwards.
38 *
39 *
40 * The current code is very much work in progress. You've been warned!
41 *
42 *
43 * @section sec_iem_fpu_instr FPU Instructions
44 *
45 * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46 * same or equivalent instructions on the host FPU. To make life easy, we also
47 * let the FPU prioritize the unmasked exceptions for us. This however, only
48 * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49 * for FPU exception delivery, because with CR0.NE=0 there is a window where we
50 * can trigger spurious FPU exceptions.
51 *
52 * The guest FPU state is not loaded into the host CPU and kept there till we
53 * leave IEM because the calling conventions have declared an all year open
54 * season on much of the FPU state. For instance an innocent looking call to
55 * memcpy might end up using a whole bunch of XMM or MM registers if the
56 * particular implementation finds it worthwhile.
57 *
58 *
59 * @section sec_iem_logging Logging
60 *
61 * The IEM code uses the \"IEM\" log group for the main logging. The different
62 * logging levels/flags are generally used for the following purposes:
63 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64 * - Flow (LogFlow): Basic enter/exit IEM state info.
65 * - Level 2 (Log2): ?
66 * - Level 3 (Log3): More detailed enter/exit IEM state info.
67 * - Level 4 (Log4): Decoding mnemonics w/ EIP.
68 * - Level 5 (Log5): Decoding details.
69 * - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
70 * - Level 7 (Log7): iret++ execution logging.
71 * - Level 8 (Log8): Memory writes.
72 * - Level 9 (Log9): Memory reads.
73 *
74 */
75
76//#define IEM_LOG_MEMORY_WRITES
77#define IEM_IMPLEMENTS_TASKSWITCH
78
79/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
80#ifdef _MSC_VER
81# pragma warning(disable:4505)
82#endif
83
84
85/*********************************************************************************************************************************
86* Header Files *
87*********************************************************************************************************************************/
88#define LOG_GROUP LOG_GROUP_IEM
89#define VMCPU_INCL_CPUM_GST_CTX
90#include <VBox/vmm/iem.h>
91#include <VBox/vmm/cpum.h>
92#include <VBox/vmm/apic.h>
93#include <VBox/vmm/pdm.h>
94#include <VBox/vmm/pgm.h>
95#include <VBox/vmm/iom.h>
96#include <VBox/vmm/em.h>
97#include <VBox/vmm/hm.h>
98#include <VBox/vmm/nem.h>
99#include <VBox/vmm/gim.h>
100#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
101# include <VBox/vmm/em.h>
102# include <VBox/vmm/hm_svm.h>
103#endif
104#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
105# include <VBox/vmm/hmvmxinline.h>
106#endif
107#include <VBox/vmm/tm.h>
108#include <VBox/vmm/dbgf.h>
109#include <VBox/vmm/dbgftrace.h>
110#include "IEMInternal.h"
111#include <VBox/vmm/vmcc.h>
112#include <VBox/log.h>
113#include <VBox/err.h>
114#include <VBox/param.h>
115#include <VBox/dis.h>
116#include <VBox/disopcode.h>
117#include <iprt/asm-math.h>
118#include <iprt/assert.h>
119#include <iprt/string.h>
120#include <iprt/x86.h>
121
122
123/*********************************************************************************************************************************
124* Structures and Typedefs *
125*********************************************************************************************************************************/
126/** @typedef PFNIEMOP
127 * Pointer to an opcode decoder function.
128 */
129
130/** @def FNIEMOP_DEF
131 * Define an opcode decoder function.
132 *
133 * We're using macors for this so that adding and removing parameters as well as
134 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
135 *
136 * @param a_Name The function name.
137 */
138
139/** @typedef PFNIEMOPRM
140 * Pointer to an opcode decoder function with RM byte.
141 */
142
143/** @def FNIEMOPRM_DEF
144 * Define an opcode decoder function with RM byte.
145 *
146 * We're using macors for this so that adding and removing parameters as well as
147 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
148 *
149 * @param a_Name The function name.
150 */
151
152#if defined(__GNUC__) && defined(RT_ARCH_X86)
153typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPUCC pVCpu);
154typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
155# define FNIEMOP_DEF(a_Name) \
156 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu)
157# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
158 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
159# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
160 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
161
162#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
163typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPUCC pVCpu);
164typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
165# define FNIEMOP_DEF(a_Name) \
166 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
167# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
168 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
169# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
170 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
171
172#elif defined(__GNUC__)
173typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
174typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
175# define FNIEMOP_DEF(a_Name) \
176 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu)
177# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
178 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0)
179# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
180 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
181
182#else
183typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPUCC pVCpu);
184typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPUCC pVCpu, uint8_t bRm);
185# define FNIEMOP_DEF(a_Name) \
186 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu) RT_NO_THROW_DEF
187# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
188 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
189# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
190 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPUCC pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
191
192#endif
193#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
194
195
196/**
197 * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
198 */
199typedef union IEMSELDESC
200{
201 /** The legacy view. */
202 X86DESC Legacy;
203 /** The long mode view. */
204 X86DESC64 Long;
205} IEMSELDESC;
206/** Pointer to a selector descriptor table entry. */
207typedef IEMSELDESC *PIEMSELDESC;
208
209/**
210 * CPU exception classes.
211 */
212typedef enum IEMXCPTCLASS
213{
214 IEMXCPTCLASS_BENIGN,
215 IEMXCPTCLASS_CONTRIBUTORY,
216 IEMXCPTCLASS_PAGE_FAULT,
217 IEMXCPTCLASS_DOUBLE_FAULT
218} IEMXCPTCLASS;
219
220
221/*********************************************************************************************************************************
222* Defined Constants And Macros *
223*********************************************************************************************************************************/
224/** @def IEM_WITH_SETJMP
225 * Enables alternative status code handling using setjmps.
226 *
227 * This adds a bit of expense via the setjmp() call since it saves all the
228 * non-volatile registers. However, it eliminates return code checks and allows
229 * for more optimal return value passing (return regs instead of stack buffer).
230 */
231#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
232# define IEM_WITH_SETJMP
233#endif
234
235/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
236 * due to GCC lacking knowledge about the value range of a switch. */
237#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
238
239/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
240#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
241
242/**
243 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
244 * occation.
245 */
246#ifdef LOG_ENABLED
247# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
248 do { \
249 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
250 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
251 } while (0)
252#else
253# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
254 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
255#endif
256
257/**
258 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
259 * occation using the supplied logger statement.
260 *
261 * @param a_LoggerArgs What to log on failure.
262 */
263#ifdef LOG_ENABLED
264# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
265 do { \
266 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
267 /*LogFunc(a_LoggerArgs);*/ \
268 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
269 } while (0)
270#else
271# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
272 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
273#endif
274
275/**
276 * Call an opcode decoder function.
277 *
278 * We're using macors for this so that adding and removing parameters can be
279 * done as we please. See FNIEMOP_DEF.
280 */
281#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
282
283/**
284 * Call a common opcode decoder function taking one extra argument.
285 *
286 * We're using macors for this so that adding and removing parameters can be
287 * done as we please. See FNIEMOP_DEF_1.
288 */
289#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
290
291/**
292 * Call a common opcode decoder function taking one extra argument.
293 *
294 * We're using macors for this so that adding and removing parameters can be
295 * done as we please. See FNIEMOP_DEF_1.
296 */
297#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
298
299/**
300 * Check if we're currently executing in real or virtual 8086 mode.
301 *
302 * @returns @c true if it is, @c false if not.
303 * @param a_pVCpu The IEM state of the current CPU.
304 */
305#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
306
307/**
308 * Check if we're currently executing in virtual 8086 mode.
309 *
310 * @returns @c true if it is, @c false if not.
311 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
312 */
313#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
314
315/**
316 * Check if we're currently executing in long mode.
317 *
318 * @returns @c true if it is, @c false if not.
319 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
320 */
321#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
322
323/**
324 * Check if we're currently executing in a 64-bit code segment.
325 *
326 * @returns @c true if it is, @c false if not.
327 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
328 */
329#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
330
331/**
332 * Check if we're currently executing in real mode.
333 *
334 * @returns @c true if it is, @c false if not.
335 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
336 */
337#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
338
339/**
340 * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
341 * @returns PCCPUMFEATURES
342 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
343 */
344#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
345
346/**
347 * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
348 * @returns PCCPUMFEATURES
349 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
350 */
351#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.HostFeatures))
352
353/**
354 * Evaluates to true if we're presenting an Intel CPU to the guest.
355 */
356#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
357
358/**
359 * Evaluates to true if we're presenting an AMD CPU to the guest.
360 */
361#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD || (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_HYGON )
362
363/**
364 * Check if the address is canonical.
365 */
366#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
367
368/**
369 * Gets the effective VEX.VVVV value.
370 *
371 * The 4th bit is ignored if not 64-bit code.
372 * @returns effective V-register value.
373 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
374 */
375#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
376 ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
377
378/** @def IEM_USE_UNALIGNED_DATA_ACCESS
379 * Use unaligned accesses instead of elaborate byte assembly. */
380#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
381# define IEM_USE_UNALIGNED_DATA_ACCESS
382#endif
383
384#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
385
386/**
387 * Check if the guest has entered VMX root operation.
388 */
389# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(a_pVCpu)))
390
391/**
392 * Check if the guest has entered VMX non-root operation.
393 */
394# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(a_pVCpu)))
395
396/**
397 * Check if the nested-guest has the given Pin-based VM-execution control set.
398 */
399# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_PinCtl) \
400 (CPUMIsGuestVmxPinCtlsSet(IEM_GET_CTX(a_pVCpu), (a_PinCtl)))
401
402/**
403 * Check if the nested-guest has the given Processor-based VM-execution control set.
404 */
405#define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_ProcCtl) \
406 (CPUMIsGuestVmxProcCtlsSet(IEM_GET_CTX(a_pVCpu), (a_ProcCtl)))
407
408/**
409 * Check if the nested-guest has the given Secondary Processor-based VM-execution
410 * control set.
411 */
412#define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_ProcCtl2) \
413 (CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(a_pVCpu), (a_ProcCtl2)))
414
415/**
416 * Invokes the VMX VM-exit handler for an instruction intercept.
417 */
418# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) \
419 do { return iemVmxVmexitInstr((a_pVCpu), (a_uExitReason), (a_cbInstr)); } while (0)
420
421/**
422 * Invokes the VMX VM-exit handler for an instruction intercept where the
423 * instruction provides additional VM-exit information.
424 */
425# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) \
426 do { return iemVmxVmexitInstrNeedsInfo((a_pVCpu), (a_uExitReason), (a_uInstrId), (a_cbInstr)); } while (0)
427
428/**
429 * Invokes the VMX VM-exit handler for a task switch.
430 */
431# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) \
432 do { return iemVmxVmexitTaskSwitch((a_pVCpu), (a_enmTaskSwitch), (a_SelNewTss), (a_cbInstr)); } while (0)
433
434/**
435 * Invokes the VMX VM-exit handler for MWAIT.
436 */
437# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) \
438 do { return iemVmxVmexitInstrMwait((a_pVCpu), (a_fMonitorArmed), (a_cbInstr)); } while (0)
439
440/**
441 * Invokes the VMX VM-exit handler for EPT faults.
442 */
443# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) \
444 do { return iemVmxVmexitEpt(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr); } while (0)
445
446/**
447 * Invokes the VMX VM-exit handler.
448 */
449# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) \
450 do { return iemVmxVmexit((a_pVCpu), (a_uExitReason), (a_uExitQual)); } while (0)
451
452#else
453# define IEM_VMX_IS_ROOT_MODE(a_pVCpu) (false)
454# define IEM_VMX_IS_NON_ROOT_MODE(a_pVCpu) (false)
455# define IEM_VMX_IS_PINCTLS_SET(a_pVCpu, a_cbInstr) (false)
456# define IEM_VMX_IS_PROCCTLS_SET(a_pVCpu, a_cbInstr) (false)
457# define IEM_VMX_IS_PROCCTLS2_SET(a_pVCpu, a_cbInstr) (false)
458# define IEM_VMX_VMEXIT_INSTR_RET(a_pVCpu, a_uExitReason, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
459# define IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(a_pVCpu, a_uExitReason, a_uInstrId, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
460# define IEM_VMX_VMEXIT_TASK_SWITCH_RET(a_pVCpu, a_enmTaskSwitch, a_SelNewTss, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
461# define IEM_VMX_VMEXIT_MWAIT_RET(a_pVCpu, a_fMonitorArmed, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
462# define IEM_VMX_VMEXIT_EPT_RET(a_pVCpu, a_pPtWalk, a_fAccess, a_fSlatFail, a_cbInstr) do { return VERR_VMX_IPE_1; } while (0)
463# define IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(a_pVCpu, a_uExitReason, a_uExitQual) do { return VERR_VMX_IPE_1; } while (0)
464
465#endif
466
467#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
468/**
469 * Check if an SVM control/instruction intercept is set.
470 */
471# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) \
472 (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
473
474/**
475 * Check if an SVM read CRx intercept is set.
476 */
477# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
478 (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
479
480/**
481 * Check if an SVM write CRx intercept is set.
482 */
483# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) \
484 (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
485
486/**
487 * Check if an SVM read DRx intercept is set.
488 */
489# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
490 (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
491
492/**
493 * Check if an SVM write DRx intercept is set.
494 */
495# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) \
496 (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
497
498/**
499 * Check if an SVM exception intercept is set.
500 */
501# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) \
502 (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
503
504/**
505 * Invokes the SVM \#VMEXIT handler for the nested-guest.
506 */
507# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
508 do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
509
510/**
511 * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
512 * corresponding decode assist information.
513 */
514# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
515 do \
516 { \
517 uint64_t uExitInfo1; \
518 if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
519 && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
520 uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
521 else \
522 uExitInfo1 = 0; \
523 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
524 } while (0)
525
526/** Check and handles SVM nested-guest instruction intercept and updates
527 * NRIP if needed.
528 */
529# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
530 do \
531 { \
532 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept)) \
533 { \
534 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
535 IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2); \
536 } \
537 } while (0)
538
539/** Checks and handles SVM nested-guest CR0 read intercept. */
540# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) \
541 do \
542 { \
543 if (!IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, 0)) \
544 { /* probably likely */ } \
545 else \
546 { \
547 IEM_SVM_UPDATE_NRIP(a_pVCpu); \
548 IEM_SVM_VMEXIT_RET(a_pVCpu, SVM_EXIT_READ_CR0, a_uExitInfo1, a_uExitInfo2); \
549 } \
550 } while (0)
551
552/**
553 * Updates the NextRIP (NRI) field in the nested-guest VMCB.
554 */
555# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
556 do { \
557 if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
558 CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
559 } while (0)
560
561#else
562# define IEM_SVM_IS_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
563# define IEM_SVM_IS_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
564# define IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
565# define IEM_SVM_IS_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
566# define IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
567# define IEM_SVM_IS_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
568# define IEM_SVM_VMEXIT_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
569# define IEM_SVM_CRX_VMEXIT_RET(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
570# define IEM_SVM_CHECK_INSTR_INTERCEPT(a_pVCpu, a_Intercept, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { } while (0)
571# define IEM_SVM_CHECK_READ_CR0_INTERCEPT(a_pVCpu, a_uExitInfo1, a_uExitInfo2) do { } while (0)
572# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
573
574#endif
575
576
577/*********************************************************************************************************************************
578* Global Variables *
579*********************************************************************************************************************************/
580extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
581
582
583/** Function table for the ADD instruction. */
584IEM_STATIC const IEMOPBINSIZES g_iemAImpl_add =
585{
586 iemAImpl_add_u8, iemAImpl_add_u8_locked,
587 iemAImpl_add_u16, iemAImpl_add_u16_locked,
588 iemAImpl_add_u32, iemAImpl_add_u32_locked,
589 iemAImpl_add_u64, iemAImpl_add_u64_locked
590};
591
592/** Function table for the ADC instruction. */
593IEM_STATIC const IEMOPBINSIZES g_iemAImpl_adc =
594{
595 iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
596 iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
597 iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
598 iemAImpl_adc_u64, iemAImpl_adc_u64_locked
599};
600
601/** Function table for the SUB instruction. */
602IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sub =
603{
604 iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
605 iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
606 iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
607 iemAImpl_sub_u64, iemAImpl_sub_u64_locked
608};
609
610/** Function table for the SBB instruction. */
611IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sbb =
612{
613 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
614 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
615 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
616 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
617};
618
619/** Function table for the OR instruction. */
620IEM_STATIC const IEMOPBINSIZES g_iemAImpl_or =
621{
622 iemAImpl_or_u8, iemAImpl_or_u8_locked,
623 iemAImpl_or_u16, iemAImpl_or_u16_locked,
624 iemAImpl_or_u32, iemAImpl_or_u32_locked,
625 iemAImpl_or_u64, iemAImpl_or_u64_locked
626};
627
628/** Function table for the XOR instruction. */
629IEM_STATIC const IEMOPBINSIZES g_iemAImpl_xor =
630{
631 iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
632 iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
633 iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
634 iemAImpl_xor_u64, iemAImpl_xor_u64_locked
635};
636
637/** Function table for the AND instruction. */
638IEM_STATIC const IEMOPBINSIZES g_iemAImpl_and =
639{
640 iemAImpl_and_u8, iemAImpl_and_u8_locked,
641 iemAImpl_and_u16, iemAImpl_and_u16_locked,
642 iemAImpl_and_u32, iemAImpl_and_u32_locked,
643 iemAImpl_and_u64, iemAImpl_and_u64_locked
644};
645
646/** Function table for the CMP instruction.
647 * @remarks Making operand order ASSUMPTIONS.
648 */
649IEM_STATIC const IEMOPBINSIZES g_iemAImpl_cmp =
650{
651 iemAImpl_cmp_u8, NULL,
652 iemAImpl_cmp_u16, NULL,
653 iemAImpl_cmp_u32, NULL,
654 iemAImpl_cmp_u64, NULL
655};
656
657/** Function table for the TEST instruction.
658 * @remarks Making operand order ASSUMPTIONS.
659 */
660IEM_STATIC const IEMOPBINSIZES g_iemAImpl_test =
661{
662 iemAImpl_test_u8, NULL,
663 iemAImpl_test_u16, NULL,
664 iemAImpl_test_u32, NULL,
665 iemAImpl_test_u64, NULL
666};
667
668/** Function table for the BT instruction. */
669IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bt =
670{
671 NULL, NULL,
672 iemAImpl_bt_u16, NULL,
673 iemAImpl_bt_u32, NULL,
674 iemAImpl_bt_u64, NULL
675};
676
677/** Function table for the BTC instruction. */
678IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btc =
679{
680 NULL, NULL,
681 iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
682 iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
683 iemAImpl_btc_u64, iemAImpl_btc_u64_locked
684};
685
686/** Function table for the BTR instruction. */
687IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btr =
688{
689 NULL, NULL,
690 iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
691 iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
692 iemAImpl_btr_u64, iemAImpl_btr_u64_locked
693};
694
695/** Function table for the BTS instruction. */
696IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bts =
697{
698 NULL, NULL,
699 iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
700 iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
701 iemAImpl_bts_u64, iemAImpl_bts_u64_locked
702};
703
704/** Function table for the BSF instruction. */
705IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf =
706{
707 NULL, NULL,
708 iemAImpl_bsf_u16, NULL,
709 iemAImpl_bsf_u32, NULL,
710 iemAImpl_bsf_u64, NULL
711};
712
713/** Function table for the BSR instruction. */
714IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr =
715{
716 NULL, NULL,
717 iemAImpl_bsr_u16, NULL,
718 iemAImpl_bsr_u32, NULL,
719 iemAImpl_bsr_u64, NULL
720};
721
722/** Function table for the IMUL instruction. */
723IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two =
724{
725 NULL, NULL,
726 iemAImpl_imul_two_u16, NULL,
727 iemAImpl_imul_two_u32, NULL,
728 iemAImpl_imul_two_u64, NULL
729};
730
731/** Group 1 /r lookup table. */
732IEM_STATIC const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
733{
734 &g_iemAImpl_add,
735 &g_iemAImpl_or,
736 &g_iemAImpl_adc,
737 &g_iemAImpl_sbb,
738 &g_iemAImpl_and,
739 &g_iemAImpl_sub,
740 &g_iemAImpl_xor,
741 &g_iemAImpl_cmp
742};
743
744/** Function table for the INC instruction. */
745IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_inc =
746{
747 iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
748 iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
749 iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
750 iemAImpl_inc_u64, iemAImpl_inc_u64_locked
751};
752
753/** Function table for the DEC instruction. */
754IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_dec =
755{
756 iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
757 iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
758 iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
759 iemAImpl_dec_u64, iemAImpl_dec_u64_locked
760};
761
762/** Function table for the NEG instruction. */
763IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_neg =
764{
765 iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
766 iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
767 iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
768 iemAImpl_neg_u64, iemAImpl_neg_u64_locked
769};
770
771/** Function table for the NOT instruction. */
772IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_not =
773{
774 iemAImpl_not_u8, iemAImpl_not_u8_locked,
775 iemAImpl_not_u16, iemAImpl_not_u16_locked,
776 iemAImpl_not_u32, iemAImpl_not_u32_locked,
777 iemAImpl_not_u64, iemAImpl_not_u64_locked
778};
779
780
781/** Function table for the ROL instruction. */
782IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol =
783{
784 iemAImpl_rol_u8,
785 iemAImpl_rol_u16,
786 iemAImpl_rol_u32,
787 iemAImpl_rol_u64
788};
789
790/** Function table for the ROR instruction. */
791IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror =
792{
793 iemAImpl_ror_u8,
794 iemAImpl_ror_u16,
795 iemAImpl_ror_u32,
796 iemAImpl_ror_u64
797};
798
799/** Function table for the RCL instruction. */
800IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl =
801{
802 iemAImpl_rcl_u8,
803 iemAImpl_rcl_u16,
804 iemAImpl_rcl_u32,
805 iemAImpl_rcl_u64
806};
807
808/** Function table for the RCR instruction. */
809IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr =
810{
811 iemAImpl_rcr_u8,
812 iemAImpl_rcr_u16,
813 iemAImpl_rcr_u32,
814 iemAImpl_rcr_u64
815};
816
817/** Function table for the SHL instruction. */
818IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl =
819{
820 iemAImpl_shl_u8,
821 iemAImpl_shl_u16,
822 iemAImpl_shl_u32,
823 iemAImpl_shl_u64
824};
825
826/** Function table for the SHR instruction. */
827IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr =
828{
829 iemAImpl_shr_u8,
830 iemAImpl_shr_u16,
831 iemAImpl_shr_u32,
832 iemAImpl_shr_u64
833};
834
835/** Function table for the SAR instruction. */
836IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar =
837{
838 iemAImpl_sar_u8,
839 iemAImpl_sar_u16,
840 iemAImpl_sar_u32,
841 iemAImpl_sar_u64
842};
843
844
845/** Function table for the MUL instruction. */
846IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul =
847{
848 iemAImpl_mul_u8,
849 iemAImpl_mul_u16,
850 iemAImpl_mul_u32,
851 iemAImpl_mul_u64
852};
853
854/** Function table for the IMUL instruction working implicitly on rAX. */
855IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul =
856{
857 iemAImpl_imul_u8,
858 iemAImpl_imul_u16,
859 iemAImpl_imul_u32,
860 iemAImpl_imul_u64
861};
862
863/** Function table for the DIV instruction. */
864IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div =
865{
866 iemAImpl_div_u8,
867 iemAImpl_div_u16,
868 iemAImpl_div_u32,
869 iemAImpl_div_u64
870};
871
872/** Function table for the MUL instruction. */
873IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv =
874{
875 iemAImpl_idiv_u8,
876 iemAImpl_idiv_u16,
877 iemAImpl_idiv_u32,
878 iemAImpl_idiv_u64
879};
880
881/** Function table for the SHLD instruction */
882IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
883{
884 iemAImpl_shld_u16,
885 iemAImpl_shld_u32,
886 iemAImpl_shld_u64,
887};
888
889/** Function table for the SHRD instruction */
890IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
891{
892 iemAImpl_shrd_u16,
893 iemAImpl_shrd_u32,
894 iemAImpl_shrd_u64,
895};
896
897
898/** Function table for the PUNPCKLBW instruction */
899IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklbw = { iemAImpl_punpcklbw_u64, iemAImpl_punpcklbw_u128 };
900/** Function table for the PUNPCKLBD instruction */
901IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklwd = { iemAImpl_punpcklwd_u64, iemAImpl_punpcklwd_u128 };
902/** Function table for the PUNPCKLDQ instruction */
903IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpckldq = { iemAImpl_punpckldq_u64, iemAImpl_punpckldq_u128 };
904/** Function table for the PUNPCKLQDQ instruction */
905IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklqdq = { NULL, iemAImpl_punpcklqdq_u128 };
906
907/** Function table for the PUNPCKHBW instruction */
908IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhbw = { iemAImpl_punpckhbw_u64, iemAImpl_punpckhbw_u128 };
909/** Function table for the PUNPCKHBD instruction */
910IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhwd = { iemAImpl_punpckhwd_u64, iemAImpl_punpckhwd_u128 };
911/** Function table for the PUNPCKHDQ instruction */
912IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhdq = { iemAImpl_punpckhdq_u64, iemAImpl_punpckhdq_u128 };
913/** Function table for the PUNPCKHQDQ instruction */
914IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhqdq = { NULL, iemAImpl_punpckhqdq_u128 };
915
916/** Function table for the PXOR instruction */
917IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pxor = { iemAImpl_pxor_u64, iemAImpl_pxor_u128 };
918/** Function table for the PCMPEQB instruction */
919IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqb = { iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqb_u128 };
920/** Function table for the PCMPEQW instruction */
921IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqw = { iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqw_u128 };
922/** Function table for the PCMPEQD instruction */
923IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqd = { iemAImpl_pcmpeqd_u64, iemAImpl_pcmpeqd_u128 };
924
925
926#if defined(IEM_LOG_MEMORY_WRITES)
927/** What IEM just wrote. */
928uint8_t g_abIemWrote[256];
929/** How much IEM just wrote. */
930size_t g_cbIemWrote;
931#endif
932
933
934/*********************************************************************************************************************************
935* Internal Functions *
936*********************************************************************************************************************************/
937IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr);
938IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu);
939IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu);
940IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel);
941/*IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);*/
942IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel);
943IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr);
944IEM_STATIC VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel);
945IEM_STATIC VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr);
946IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr);
947IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu);
948IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL uSel);
949IEM_STATIC VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
950IEM_STATIC VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel);
951IEM_STATIC VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
952IEM_STATIC VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
953IEM_STATIC VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu);
954#ifdef IEM_WITH_SETJMP
955DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
956DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu);
957DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
958DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel);
959DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess);
960#endif
961
962IEM_STATIC VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
963IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess);
964IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
965IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
966IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
967IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
968IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
969IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
970IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
971IEM_STATIC VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode);
972IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt);
973IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp);
974IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp);
975IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value);
976IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value);
977IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel);
978DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg);
979DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg);
980
981#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
982IEM_STATIC VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual);
983IEM_STATIC VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr);
984IEM_STATIC VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode, uint64_t uCr2, uint8_t cbInstr);
985IEM_STATIC VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu);
986IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMem(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, void *pvData, uint32_t fAccess);
987IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value);
988IEM_STATIC VBOXSTRICTRC iemVmxVirtApicAccessMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Value);
989IEM_STATIC VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr);
990#endif
991
992#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
993IEM_STATIC VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2);
994IEM_STATIC VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2);
995#endif
996
997
998/**
999 * Sets the pass up status.
1000 *
1001 * @returns VINF_SUCCESS.
1002 * @param pVCpu The cross context virtual CPU structure of the
1003 * calling thread.
1004 * @param rcPassUp The pass up status. Must be informational.
1005 * VINF_SUCCESS is not allowed.
1006 */
1007IEM_STATIC int iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
1008{
1009 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
1010
1011 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
1012 if (rcOldPassUp == VINF_SUCCESS)
1013 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
1014 /* If both are EM scheduling codes, use EM priority rules. */
1015 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
1016 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
1017 {
1018 if (rcPassUp < rcOldPassUp)
1019 {
1020 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1021 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
1022 }
1023 else
1024 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1025 }
1026 /* Override EM scheduling with specific status code. */
1027 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
1028 {
1029 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1030 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
1031 }
1032 /* Don't override specific status code, first come first served. */
1033 else
1034 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
1035 return VINF_SUCCESS;
1036}
1037
1038
1039/**
1040 * Calculates the CPU mode.
1041 *
1042 * This is mainly for updating IEMCPU::enmCpuMode.
1043 *
1044 * @returns CPU mode.
1045 * @param pVCpu The cross context virtual CPU structure of the
1046 * calling thread.
1047 */
1048DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
1049{
1050 if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
1051 return IEMMODE_64BIT;
1052 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
1053 return IEMMODE_32BIT;
1054 return IEMMODE_16BIT;
1055}
1056
1057
1058/**
1059 * Initializes the execution state.
1060 *
1061 * @param pVCpu The cross context virtual CPU structure of the
1062 * calling thread.
1063 * @param fBypassHandlers Whether to bypass access handlers.
1064 *
1065 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
1066 * side-effects in strict builds.
1067 */
1068DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
1069{
1070 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
1071 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1072 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1073 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1074 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1075 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1076 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1077 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1078 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1079 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1080
1081 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
1082 pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1083#ifdef VBOX_STRICT
1084 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
1085 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
1086 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
1087 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
1088 pVCpu->iem.s.fPrefixes = 0xfeedbeef;
1089 pVCpu->iem.s.uRexReg = 127;
1090 pVCpu->iem.s.uRexB = 127;
1091 pVCpu->iem.s.offModRm = 127;
1092 pVCpu->iem.s.uRexIndex = 127;
1093 pVCpu->iem.s.iEffSeg = 127;
1094 pVCpu->iem.s.idxPrefix = 127;
1095 pVCpu->iem.s.uVex3rdReg = 127;
1096 pVCpu->iem.s.uVexLength = 127;
1097 pVCpu->iem.s.fEvexStuff = 127;
1098 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
1099# ifdef IEM_WITH_CODE_TLB
1100 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
1101 pVCpu->iem.s.pbInstrBuf = NULL;
1102 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
1103 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
1104 pVCpu->iem.s.offCurInstrStart = INT16_MAX;
1105 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
1106# else
1107 pVCpu->iem.s.offOpcode = 127;
1108 pVCpu->iem.s.cbOpcode = 127;
1109# endif
1110#endif
1111
1112 pVCpu->iem.s.cActiveMappings = 0;
1113 pVCpu->iem.s.iNextMapping = 0;
1114 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1115 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
1116#if 0
1117#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1118 if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
1119 && CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
1120 {
1121 PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
1122 Assert(pVmcs);
1123 RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
1124 if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
1125 {
1126 int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
1127 pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
1128 NIL_RTR0PTR /* pvUserR0 */, NIL_RTRCPTR /* pvUserRC */, NULL /* pszDesc */);
1129 AssertRC(rc);
1130 }
1131 }
1132#endif
1133#endif
1134}
1135
1136#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
1137/**
1138 * Performs a minimal reinitialization of the execution state.
1139 *
1140 * This is intended to be used by VM-exits, SMM, LOADALL and other similar
1141 * 'world-switch' types operations on the CPU. Currently only nested
1142 * hardware-virtualization uses it.
1143 *
1144 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1145 */
1146IEM_STATIC void iemReInitExec(PVMCPUCC pVCpu)
1147{
1148 IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
1149 uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
1150
1151 pVCpu->iem.s.uCpl = uCpl;
1152 pVCpu->iem.s.enmCpuMode = enmMode;
1153 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1154 pVCpu->iem.s.enmEffAddrMode = enmMode;
1155 if (enmMode != IEMMODE_64BIT)
1156 {
1157 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1158 pVCpu->iem.s.enmEffOpSize = enmMode;
1159 }
1160 else
1161 {
1162 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1163 pVCpu->iem.s.enmEffOpSize = enmMode;
1164 }
1165 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1166#ifndef IEM_WITH_CODE_TLB
1167 /** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
1168 pVCpu->iem.s.offOpcode = 0;
1169 pVCpu->iem.s.cbOpcode = 0;
1170#endif
1171 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1172}
1173#endif
1174
1175/**
1176 * Counterpart to #iemInitExec that undoes evil strict-build stuff.
1177 *
1178 * @param pVCpu The cross context virtual CPU structure of the
1179 * calling thread.
1180 */
1181DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
1182{
1183 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
1184#ifdef VBOX_STRICT
1185# ifdef IEM_WITH_CODE_TLB
1186 NOREF(pVCpu);
1187# else
1188 pVCpu->iem.s.cbOpcode = 0;
1189# endif
1190#else
1191 NOREF(pVCpu);
1192#endif
1193}
1194
1195
1196/**
1197 * Initializes the decoder state.
1198 *
1199 * iemReInitDecoder is mostly a copy of this function.
1200 *
1201 * @param pVCpu The cross context virtual CPU structure of the
1202 * calling thread.
1203 * @param fBypassHandlers Whether to bypass access handlers.
1204 * @param fDisregardLock Whether to disregard the LOCK prefix.
1205 */
1206DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
1207{
1208 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
1209 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1210 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1211 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1212 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1213 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1214 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1215 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1216 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1217 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1218
1219 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
1220 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
1221 pVCpu->iem.s.enmCpuMode = enmMode;
1222 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1223 pVCpu->iem.s.enmEffAddrMode = enmMode;
1224 if (enmMode != IEMMODE_64BIT)
1225 {
1226 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1227 pVCpu->iem.s.enmEffOpSize = enmMode;
1228 }
1229 else
1230 {
1231 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1232 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
1233 }
1234 pVCpu->iem.s.fPrefixes = 0;
1235 pVCpu->iem.s.uRexReg = 0;
1236 pVCpu->iem.s.uRexB = 0;
1237 pVCpu->iem.s.uRexIndex = 0;
1238 pVCpu->iem.s.idxPrefix = 0;
1239 pVCpu->iem.s.uVex3rdReg = 0;
1240 pVCpu->iem.s.uVexLength = 0;
1241 pVCpu->iem.s.fEvexStuff = 0;
1242 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1243#ifdef IEM_WITH_CODE_TLB
1244 pVCpu->iem.s.pbInstrBuf = NULL;
1245 pVCpu->iem.s.offInstrNextByte = 0;
1246 pVCpu->iem.s.offCurInstrStart = 0;
1247# ifdef VBOX_STRICT
1248 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
1249 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
1250 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
1251# endif
1252#else
1253 pVCpu->iem.s.offOpcode = 0;
1254 pVCpu->iem.s.cbOpcode = 0;
1255#endif
1256 pVCpu->iem.s.offModRm = 0;
1257 pVCpu->iem.s.cActiveMappings = 0;
1258 pVCpu->iem.s.iNextMapping = 0;
1259 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1260 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
1261 pVCpu->iem.s.fDisregardLock = fDisregardLock;
1262
1263#ifdef DBGFTRACE_ENABLED
1264 switch (enmMode)
1265 {
1266 case IEMMODE_64BIT:
1267 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
1268 break;
1269 case IEMMODE_32BIT:
1270 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1271 break;
1272 case IEMMODE_16BIT:
1273 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1274 break;
1275 }
1276#endif
1277}
1278
1279
1280/**
1281 * Reinitializes the decoder state 2nd+ loop of IEMExecLots.
1282 *
1283 * This is mostly a copy of iemInitDecoder.
1284 *
1285 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1286 */
1287DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
1288{
1289 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1290 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1291 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1292 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1293 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1294 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1295 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1296 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1297 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1298
1299 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
1300 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
1301 pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
1302 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1303 pVCpu->iem.s.enmEffAddrMode = enmMode;
1304 if (enmMode != IEMMODE_64BIT)
1305 {
1306 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1307 pVCpu->iem.s.enmEffOpSize = enmMode;
1308 }
1309 else
1310 {
1311 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1312 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
1313 }
1314 pVCpu->iem.s.fPrefixes = 0;
1315 pVCpu->iem.s.uRexReg = 0;
1316 pVCpu->iem.s.uRexB = 0;
1317 pVCpu->iem.s.uRexIndex = 0;
1318 pVCpu->iem.s.idxPrefix = 0;
1319 pVCpu->iem.s.uVex3rdReg = 0;
1320 pVCpu->iem.s.uVexLength = 0;
1321 pVCpu->iem.s.fEvexStuff = 0;
1322 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1323#ifdef IEM_WITH_CODE_TLB
1324 if (pVCpu->iem.s.pbInstrBuf)
1325 {
1326 uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
1327 - pVCpu->iem.s.uInstrBufPc;
1328 if (off < pVCpu->iem.s.cbInstrBufTotal)
1329 {
1330 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1331 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1332 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1333 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1334 else
1335 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1336 }
1337 else
1338 {
1339 pVCpu->iem.s.pbInstrBuf = NULL;
1340 pVCpu->iem.s.offInstrNextByte = 0;
1341 pVCpu->iem.s.offCurInstrStart = 0;
1342 pVCpu->iem.s.cbInstrBuf = 0;
1343 pVCpu->iem.s.cbInstrBufTotal = 0;
1344 }
1345 }
1346 else
1347 {
1348 pVCpu->iem.s.offInstrNextByte = 0;
1349 pVCpu->iem.s.offCurInstrStart = 0;
1350 pVCpu->iem.s.cbInstrBuf = 0;
1351 pVCpu->iem.s.cbInstrBufTotal = 0;
1352 }
1353#else
1354 pVCpu->iem.s.cbOpcode = 0;
1355 pVCpu->iem.s.offOpcode = 0;
1356#endif
1357 pVCpu->iem.s.offModRm = 0;
1358 Assert(pVCpu->iem.s.cActiveMappings == 0);
1359 pVCpu->iem.s.iNextMapping = 0;
1360 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
1361 Assert(pVCpu->iem.s.fBypassHandlers == false);
1362
1363#ifdef DBGFTRACE_ENABLED
1364 switch (enmMode)
1365 {
1366 case IEMMODE_64BIT:
1367 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
1368 break;
1369 case IEMMODE_32BIT:
1370 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1371 break;
1372 case IEMMODE_16BIT:
1373 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1374 break;
1375 }
1376#endif
1377}
1378
1379
1380
1381/**
1382 * Prefetch opcodes the first time when starting executing.
1383 *
1384 * @returns Strict VBox status code.
1385 * @param pVCpu The cross context virtual CPU structure of the
1386 * calling thread.
1387 * @param fBypassHandlers Whether to bypass access handlers.
1388 * @param fDisregardLock Whether to disregard LOCK prefixes.
1389 *
1390 * @todo Combine fDisregardLock and fBypassHandlers into a flag parameter and
1391 * store them as such.
1392 */
1393IEM_STATIC VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
1394{
1395 iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock);
1396
1397#ifdef IEM_WITH_CODE_TLB
1398 /** @todo Do ITLB lookup here. */
1399
1400#else /* !IEM_WITH_CODE_TLB */
1401
1402 /*
1403 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1404 *
1405 * First translate CS:rIP to a physical address.
1406 */
1407 uint32_t cbToTryRead;
1408 RTGCPTR GCPtrPC;
1409 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1410 {
1411 cbToTryRead = GUEST_PAGE_SIZE;
1412 GCPtrPC = pVCpu->cpum.GstCtx.rip;
1413 if (IEM_IS_CANONICAL(GCPtrPC))
1414 cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
1415 else
1416 return iemRaiseGeneralProtectionFault0(pVCpu);
1417 }
1418 else
1419 {
1420 uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
1421 AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1422 if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
1423 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
1424 else
1425 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1426 if (cbToTryRead) { /* likely */ }
1427 else /* overflowed */
1428 {
1429 Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1430 cbToTryRead = UINT32_MAX;
1431 }
1432 GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
1433 Assert(GCPtrPC <= UINT32_MAX);
1434 }
1435
1436 PGMPTWALK Walk;
1437 int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
1438 if (RT_SUCCESS(rc))
1439 Assert(Walk.fSucceeded); /* probable. */
1440 else
1441 {
1442 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
1443#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1444 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1445 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1446#endif
1447 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
1448 }
1449 if ((Walk.fEffective & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ }
1450 else
1451 {
1452 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
1453#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1454 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1455 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1456#endif
1457 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1458 }
1459 if (!(Walk.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
1460 else
1461 {
1462 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
1463#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1464 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1465 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
1466#endif
1467 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1468 }
1469 RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
1470 /** @todo Check reserved bits and such stuff. PGM is better at doing
1471 * that, so do it when implementing the guest virtual address
1472 * TLB... */
1473
1474 /*
1475 * Read the bytes at this address.
1476 */
1477 uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
1478 if (cbToTryRead > cbLeftOnPage)
1479 cbToTryRead = cbLeftOnPage;
1480 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
1481 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
1482
1483 if (!pVCpu->iem.s.fBypassHandlers)
1484 {
1485 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
1486 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1487 { /* likely */ }
1488 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1489 {
1490 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1491 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1492 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1493 }
1494 else
1495 {
1496 Log((RT_SUCCESS(rcStrict)
1497 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1498 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1499 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1500 return rcStrict;
1501 }
1502 }
1503 else
1504 {
1505 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
1506 if (RT_SUCCESS(rc))
1507 { /* likely */ }
1508 else
1509 {
1510 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
1511 GCPtrPC, GCPhys, rc, cbToTryRead));
1512 return rc;
1513 }
1514 }
1515 pVCpu->iem.s.cbOpcode = cbToTryRead;
1516#endif /* !IEM_WITH_CODE_TLB */
1517 return VINF_SUCCESS;
1518}
1519
1520
1521/**
1522 * Invalidates the IEM TLBs.
1523 *
1524 * This is called internally as well as by PGM when moving GC mappings.
1525 *
1526 * @returns
1527 * @param pVCpu The cross context virtual CPU structure of the calling
1528 * thread.
1529 * @param fVmm Set when PGM calls us with a remapping.
1530 */
1531VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu, bool fVmm)
1532{
1533#ifdef IEM_WITH_CODE_TLB
1534 pVCpu->iem.s.cbInstrBufTotal = 0;
1535 pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
1536 if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
1537 { /* very likely */ }
1538 else
1539 {
1540 pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
1541 unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
1542 while (i-- > 0)
1543 pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
1544 }
1545#endif
1546
1547#ifdef IEM_WITH_DATA_TLB
1548 pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
1549 if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
1550 { /* very likely */ }
1551 else
1552 {
1553 pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
1554 unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
1555 while (i-- > 0)
1556 pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
1557 }
1558#endif
1559 NOREF(pVCpu); NOREF(fVmm);
1560}
1561
1562
1563/**
1564 * Invalidates a page in the TLBs.
1565 *
1566 * @param pVCpu The cross context virtual CPU structure of the calling
1567 * thread.
1568 * @param GCPtr The address of the page to invalidate
1569 */
1570VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
1571{
1572#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
1573 GCPtr = GCPtr >> X86_PAGE_SHIFT;
1574 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
1575 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries) == 256);
1576 uintptr_t idx = (uint8_t)GCPtr;
1577
1578# ifdef IEM_WITH_CODE_TLB
1579 if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision))
1580 {
1581 pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
1582 if (GCPtr == (pVCpu->iem.s.uInstrBufPc >> X86_PAGE_SHIFT))
1583 pVCpu->iem.s.cbInstrBufTotal = 0;
1584 }
1585# endif
1586
1587# ifdef IEM_WITH_DATA_TLB
1588 if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision))
1589 pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
1590# endif
1591#else
1592 NOREF(pVCpu); NOREF(GCPtr);
1593#endif
1594}
1595
1596
1597/**
1598 * Invalidates the host physical aspects of the IEM TLBs.
1599 *
1600 * This is called internally as well as by PGM when moving GC mappings.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure of the calling
1603 * thread.
1604 */
1605VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
1606{
1607#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
1608 /* Note! This probably won't end up looking exactly like this, but it give an idea... */
1609
1610# ifdef IEM_WITH_CODE_TLB
1611 pVCpu->iem.s.cbInstrBufTotal = 0;
1612# endif
1613 uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
1614 if (uTlbPhysRev != 0)
1615 {
1616 pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
1617 pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
1618 }
1619 else
1620 {
1621 pVCpu->iem.s.CodeTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
1622 pVCpu->iem.s.DataTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
1623
1624 unsigned i;
1625# ifdef IEM_WITH_CODE_TLB
1626 i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
1627 while (i-- > 0)
1628 {
1629 pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
1630 pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
1631 }
1632# endif
1633# ifdef IEM_WITH_DATA_TLB
1634 i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
1635 while (i-- > 0)
1636 {
1637 pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
1638 pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
1639 }
1640# endif
1641 }
1642#else
1643 NOREF(pVCpu);
1644#endif
1645}
1646
1647
1648/**
1649 * Invalidates the host physical aspects of the IEM TLBs.
1650 *
1651 * This is called internally as well as by PGM when moving GC mappings.
1652 *
1653 * @param pVM The cross context VM structure.
1654 *
1655 * @remarks Caller holds the PGM lock.
1656 */
1657VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVM pVM)
1658{
1659 RT_NOREF_PV(pVM);
1660}
1661
1662#ifdef IEM_WITH_CODE_TLB
1663
1664/**
1665 * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
1666 * failure and jumps.
1667 *
1668 * We end up here for a number of reasons:
1669 * - pbInstrBuf isn't yet initialized.
1670 * - Advancing beyond the buffer boundrary (e.g. cross page).
1671 * - Advancing beyond the CS segment limit.
1672 * - Fetching from non-mappable page (e.g. MMIO).
1673 *
1674 * @param pVCpu The cross context virtual CPU structure of the
1675 * calling thread.
1676 * @param pvDst Where to return the bytes.
1677 * @param cbDst Number of bytes to read.
1678 *
1679 * @todo Make cbDst = 0 a way of initializing pbInstrBuf?
1680 */
1681IEM_STATIC void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst)
1682{
1683#ifdef IN_RING3
1684 for (;;)
1685 {
1686 Assert(cbDst <= 8);
1687 uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
1688
1689 /*
1690 * We might have a partial buffer match, deal with that first to make the
1691 * rest simpler. This is the first part of the cross page/buffer case.
1692 */
1693 if (pVCpu->iem.s.pbInstrBuf != NULL)
1694 {
1695 if (offBuf < pVCpu->iem.s.cbInstrBuf)
1696 {
1697 Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
1698 uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
1699 memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
1700
1701 cbDst -= cbCopy;
1702 pvDst = (uint8_t *)pvDst + cbCopy;
1703 offBuf += cbCopy;
1704 pVCpu->iem.s.offInstrNextByte += offBuf;
1705 }
1706 }
1707
1708 /*
1709 * Check segment limit, figuring how much we're allowed to access at this point.
1710 *
1711 * We will fault immediately if RIP is past the segment limit / in non-canonical
1712 * territory. If we do continue, there are one or more bytes to read before we
1713 * end up in trouble and we need to do that first before faulting.
1714 */
1715 RTGCPTR GCPtrFirst;
1716 uint32_t cbMaxRead;
1717 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1718 {
1719 GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
1720 if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
1721 { /* likely */ }
1722 else
1723 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
1724 cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
1725 }
1726 else
1727 {
1728 GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
1729 Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1730 if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
1731 { /* likely */ }
1732 else
1733 iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1734 cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
1735 if (cbMaxRead != 0)
1736 { /* likely */ }
1737 else
1738 {
1739 /* Overflowed because address is 0 and limit is max. */
1740 Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1741 cbMaxRead = X86_PAGE_SIZE;
1742 }
1743 GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
1744 uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
1745 if (cbMaxRead2 < cbMaxRead)
1746 cbMaxRead = cbMaxRead2;
1747 /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
1748 }
1749
1750 /*
1751 * Get the TLB entry for this piece of code.
1752 */
1753 uint64_t uTag = (GCPtrFirst >> X86_PAGE_SHIFT) | pVCpu->iem.s.CodeTlb.uTlbRevision;
1754 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
1755 PIEMTLBENTRY pTlbe = &pVCpu->iem.s.CodeTlb.aEntries[(uint8_t)uTag];
1756 if (pTlbe->uTag == uTag)
1757 {
1758 /* likely when executing lots of code, otherwise unlikely */
1759# ifdef VBOX_WITH_STATISTICS
1760 pVCpu->iem.s.CodeTlb.cTlbHits++;
1761# endif
1762 }
1763 else
1764 {
1765 pVCpu->iem.s.CodeTlb.cTlbMisses++;
1766 PGMPTWALK Walk;
1767 int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
1768 if (RT_FAILURE(rc))
1769 {
1770#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1771 /** @todo Nested VMX: Need to handle EPT violation/misconfig here? */
1772 Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
1773#endif
1774 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
1775 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
1776 }
1777
1778 AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
1779 Assert(Walk.fSucceeded);
1780 pTlbe->uTag = uTag;
1781 pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D))
1782 | (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
1783 pTlbe->GCPhys = Walk.GCPhys;
1784 pTlbe->pbMappingR3 = NULL;
1785 }
1786
1787 /*
1788 * Check TLB page table level access flags.
1789 */
1790 if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
1791 {
1792 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
1793 {
1794 Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
1795 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1796 }
1797 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1798 {
1799 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
1800 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1801 }
1802 }
1803
1804 /*
1805 * Look up the physical page info if necessary.
1806 */
1807 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1808 { /* not necessary */ }
1809 else
1810 {
1811 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
1812 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
1813 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
1814 pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
1815 | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE);
1816 int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
1817 &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
1818 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
1819 }
1820
1821# if defined(IN_RING3) || defined(IN_RING0) /** @todo fixme */
1822 /*
1823 * Try do a direct read using the pbMappingR3 pointer.
1824 */
1825 if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
1826 == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1827 {
1828 uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
1829 pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
1830 if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
1831 {
1832 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
1833 pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
1834 }
1835 else
1836 {
1837 uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
1838 Assert(cbInstr < cbMaxRead);
1839 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
1840 pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
1841 }
1842 if (cbDst <= cbMaxRead)
1843 {
1844 pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
1845 pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
1846 pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
1847 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
1848 return;
1849 }
1850 pVCpu->iem.s.pbInstrBuf = NULL;
1851
1852 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
1853 pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
1854 }
1855 else
1856# endif
1857#if 0
1858 /*
1859 * If there is no special read handling, so we can read a bit more and
1860 * put it in the prefetch buffer.
1861 */
1862 if ( cbDst < cbMaxRead
1863 && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1864 {
1865 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
1866 &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
1867 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1868 { /* likely */ }
1869 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1870 {
1871 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1872 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1873 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1874 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
1875 }
1876 else
1877 {
1878 Log((RT_SUCCESS(rcStrict)
1879 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1880 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1881 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1882 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1883 }
1884 }
1885 /*
1886 * Special read handling, so only read exactly what's needed.
1887 * This is a highly unlikely scenario.
1888 */
1889 else
1890#endif
1891 {
1892 pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
1893 uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
1894 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
1895 pvDst, cbToRead, PGMACCESSORIGIN_IEM);
1896 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1897 { /* likely */ }
1898 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1899 {
1900 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1901 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1902 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1903 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
1904 }
1905 else
1906 {
1907 Log((RT_SUCCESS(rcStrict)
1908 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1909 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1910 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1911 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1912 }
1913 pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
1914 if (cbToRead == cbDst)
1915 return;
1916 }
1917
1918 /*
1919 * More to read, loop.
1920 */
1921 cbDst -= cbMaxRead;
1922 pvDst = (uint8_t *)pvDst + cbMaxRead;
1923 }
1924#else
1925 RT_NOREF(pvDst, cbDst);
1926 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
1927#endif
1928}
1929
1930#else
1931
1932/**
1933 * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
1934 * exception if it fails.
1935 *
1936 * @returns Strict VBox status code.
1937 * @param pVCpu The cross context virtual CPU structure of the
1938 * calling thread.
1939 * @param cbMin The minimum number of bytes relative offOpcode
1940 * that must be read.
1941 */
1942IEM_STATIC VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin)
1943{
1944 /*
1945 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1946 *
1947 * First translate CS:rIP to a physical address.
1948 */
1949 uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
1950 uint32_t cbToTryRead;
1951 RTGCPTR GCPtrNext;
1952 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1953 {
1954 cbToTryRead = GUEST_PAGE_SIZE;
1955 GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
1956 if (!IEM_IS_CANONICAL(GCPtrNext))
1957 return iemRaiseGeneralProtectionFault0(pVCpu);
1958 }
1959 else
1960 {
1961 uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
1962 Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1963 GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1964 if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1965 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1966 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1967 if (!cbToTryRead) /* overflowed */
1968 {
1969 Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1970 cbToTryRead = UINT32_MAX;
1971 /** @todo check out wrapping around the code segment. */
1972 }
1973 if (cbToTryRead < cbMin - cbLeft)
1974 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1975 GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1976 }
1977
1978 /* Only read up to the end of the page, and make sure we don't read more
1979 than the opcode buffer can hold. */
1980 uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
1981 if (cbToTryRead > cbLeftOnPage)
1982 cbToTryRead = cbLeftOnPage;
1983 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
1984 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
1985/** @todo r=bird: Convert assertion into undefined opcode exception? */
1986 Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
1987
1988 PGMPTWALK Walk;
1989 int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
1990 if (RT_FAILURE(rc))
1991 {
1992 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
1993#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1994 if (Walk.fFailed & PGM_WALKFAIL_EPT)
1995 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
1996#endif
1997 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
1998 }
1999 if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
2000 {
2001 Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
2002#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2003 if (Walk.fFailed & PGM_WALKFAIL_EPT)
2004 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
2005#endif
2006 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
2007 }
2008 if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
2009 {
2010 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
2011#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2012 if (Walk.fFailed & PGM_WALKFAIL_EPT)
2013 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
2014#endif
2015 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
2016 }
2017 RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
2018 Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
2019 /** @todo Check reserved bits and such stuff. PGM is better at doing
2020 * that, so do it when implementing the guest virtual address
2021 * TLB... */
2022
2023 /*
2024 * Read the bytes at this address.
2025 *
2026 * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
2027 * and since PATM should only patch the start of an instruction there
2028 * should be no need to check again here.
2029 */
2030 if (!pVCpu->iem.s.fBypassHandlers)
2031 {
2032 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
2033 cbToTryRead, PGMACCESSORIGIN_IEM);
2034 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2035 { /* likely */ }
2036 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2037 {
2038 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
2039 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
2040 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
2041 }
2042 else
2043 {
2044 Log((RT_SUCCESS(rcStrict)
2045 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
2046 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
2047 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
2048 return rcStrict;
2049 }
2050 }
2051 else
2052 {
2053 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
2054 if (RT_SUCCESS(rc))
2055 { /* likely */ }
2056 else
2057 {
2058 Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
2059 return rc;
2060 }
2061 }
2062 pVCpu->iem.s.cbOpcode += cbToTryRead;
2063 Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
2064
2065 return VINF_SUCCESS;
2066}
2067
2068#endif /* !IEM_WITH_CODE_TLB */
2069#ifndef IEM_WITH_SETJMP
2070
2071/**
2072 * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
2073 *
2074 * @returns Strict VBox status code.
2075 * @param pVCpu The cross context virtual CPU structure of the
2076 * calling thread.
2077 * @param pb Where to return the opcode byte.
2078 */
2079DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb)
2080{
2081 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
2082 if (rcStrict == VINF_SUCCESS)
2083 {
2084 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2085 *pb = pVCpu->iem.s.abOpcode[offOpcode];
2086 pVCpu->iem.s.offOpcode = offOpcode + 1;
2087 }
2088 else
2089 *pb = 0;
2090 return rcStrict;
2091}
2092
2093
2094/**
2095 * Fetches the next opcode byte.
2096 *
2097 * @returns Strict VBox status code.
2098 * @param pVCpu The cross context virtual CPU structure of the
2099 * calling thread.
2100 * @param pu8 Where to return the opcode byte.
2101 */
2102DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
2103{
2104 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2105 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2106 {
2107 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2108 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
2109 return VINF_SUCCESS;
2110 }
2111 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
2112}
2113
2114#else /* IEM_WITH_SETJMP */
2115
2116/**
2117 * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
2118 *
2119 * @returns The opcode byte.
2120 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2121 */
2122DECL_NO_INLINE(IEM_STATIC, uint8_t) iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu)
2123{
2124# ifdef IEM_WITH_CODE_TLB
2125 uint8_t u8;
2126 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
2127 return u8;
2128# else
2129 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
2130 if (rcStrict == VINF_SUCCESS)
2131 return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
2132 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2133# endif
2134}
2135
2136
2137/**
2138 * Fetches the next opcode byte, longjmp on error.
2139 *
2140 * @returns The opcode byte.
2141 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2142 */
2143DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
2144{
2145# ifdef IEM_WITH_CODE_TLB
2146 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2147 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2148 if (RT_LIKELY( pbBuf != NULL
2149 && offBuf < pVCpu->iem.s.cbInstrBuf))
2150 {
2151 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
2152 return pbBuf[offBuf];
2153 }
2154# else
2155 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
2156 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2157 {
2158 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2159 return pVCpu->iem.s.abOpcode[offOpcode];
2160 }
2161# endif
2162 return iemOpcodeGetNextU8SlowJmp(pVCpu);
2163}
2164
2165#endif /* IEM_WITH_SETJMP */
2166
2167/**
2168 * Fetches the next opcode byte, returns automatically on failure.
2169 *
2170 * @param a_pu8 Where to return the opcode byte.
2171 * @remark Implicitly references pVCpu.
2172 */
2173#ifndef IEM_WITH_SETJMP
2174# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
2175 do \
2176 { \
2177 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
2178 if (rcStrict2 == VINF_SUCCESS) \
2179 { /* likely */ } \
2180 else \
2181 return rcStrict2; \
2182 } while (0)
2183#else
2184# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
2185#endif /* IEM_WITH_SETJMP */
2186
2187
2188#ifndef IEM_WITH_SETJMP
2189/**
2190 * Fetches the next signed byte from the opcode stream.
2191 *
2192 * @returns Strict VBox status code.
2193 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2194 * @param pi8 Where to return the signed byte.
2195 */
2196DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
2197{
2198 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
2199}
2200#endif /* !IEM_WITH_SETJMP */
2201
2202
2203/**
2204 * Fetches the next signed byte from the opcode stream, returning automatically
2205 * on failure.
2206 *
2207 * @param a_pi8 Where to return the signed byte.
2208 * @remark Implicitly references pVCpu.
2209 */
2210#ifndef IEM_WITH_SETJMP
2211# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
2212 do \
2213 { \
2214 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
2215 if (rcStrict2 != VINF_SUCCESS) \
2216 return rcStrict2; \
2217 } while (0)
2218#else /* IEM_WITH_SETJMP */
2219# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2220
2221#endif /* IEM_WITH_SETJMP */
2222
2223#ifndef IEM_WITH_SETJMP
2224
2225/**
2226 * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
2227 *
2228 * @returns Strict VBox status code.
2229 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2230 * @param pu16 Where to return the opcode dword.
2231 */
2232DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16)
2233{
2234 uint8_t u8;
2235 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2236 if (rcStrict == VINF_SUCCESS)
2237 *pu16 = (int8_t)u8;
2238 return rcStrict;
2239}
2240
2241
2242/**
2243 * Fetches the next signed byte from the opcode stream, extending it to
2244 * unsigned 16-bit.
2245 *
2246 * @returns Strict VBox status code.
2247 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2248 * @param pu16 Where to return the unsigned word.
2249 */
2250DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
2251{
2252 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2253 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2254 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
2255
2256 *pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2257 pVCpu->iem.s.offOpcode = offOpcode + 1;
2258 return VINF_SUCCESS;
2259}
2260
2261#endif /* !IEM_WITH_SETJMP */
2262
2263/**
2264 * Fetches the next signed byte from the opcode stream and sign-extending it to
2265 * a word, returning automatically on failure.
2266 *
2267 * @param a_pu16 Where to return the word.
2268 * @remark Implicitly references pVCpu.
2269 */
2270#ifndef IEM_WITH_SETJMP
2271# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
2272 do \
2273 { \
2274 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
2275 if (rcStrict2 != VINF_SUCCESS) \
2276 return rcStrict2; \
2277 } while (0)
2278#else
2279# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2280#endif
2281
2282#ifndef IEM_WITH_SETJMP
2283
2284/**
2285 * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
2286 *
2287 * @returns Strict VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2289 * @param pu32 Where to return the opcode dword.
2290 */
2291DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
2292{
2293 uint8_t u8;
2294 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2295 if (rcStrict == VINF_SUCCESS)
2296 *pu32 = (int8_t)u8;
2297 return rcStrict;
2298}
2299
2300
2301/**
2302 * Fetches the next signed byte from the opcode stream, extending it to
2303 * unsigned 32-bit.
2304 *
2305 * @returns Strict VBox status code.
2306 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2307 * @param pu32 Where to return the unsigned dword.
2308 */
2309DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
2310{
2311 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2312 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2313 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
2314
2315 *pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2316 pVCpu->iem.s.offOpcode = offOpcode + 1;
2317 return VINF_SUCCESS;
2318}
2319
2320#endif /* !IEM_WITH_SETJMP */
2321
2322/**
2323 * Fetches the next signed byte from the opcode stream and sign-extending it to
2324 * a word, returning automatically on failure.
2325 *
2326 * @param a_pu32 Where to return the word.
2327 * @remark Implicitly references pVCpu.
2328 */
2329#ifndef IEM_WITH_SETJMP
2330#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
2331 do \
2332 { \
2333 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
2334 if (rcStrict2 != VINF_SUCCESS) \
2335 return rcStrict2; \
2336 } while (0)
2337#else
2338# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2339#endif
2340
2341#ifndef IEM_WITH_SETJMP
2342
2343/**
2344 * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
2345 *
2346 * @returns Strict VBox status code.
2347 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2348 * @param pu64 Where to return the opcode qword.
2349 */
2350DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
2351{
2352 uint8_t u8;
2353 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2354 if (rcStrict == VINF_SUCCESS)
2355 *pu64 = (int8_t)u8;
2356 return rcStrict;
2357}
2358
2359
2360/**
2361 * Fetches the next signed byte from the opcode stream, extending it to
2362 * unsigned 64-bit.
2363 *
2364 * @returns Strict VBox status code.
2365 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2366 * @param pu64 Where to return the unsigned qword.
2367 */
2368DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
2369{
2370 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2371 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2372 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
2373
2374 *pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2375 pVCpu->iem.s.offOpcode = offOpcode + 1;
2376 return VINF_SUCCESS;
2377}
2378
2379#endif /* !IEM_WITH_SETJMP */
2380
2381
2382/**
2383 * Fetches the next signed byte from the opcode stream and sign-extending it to
2384 * a word, returning automatically on failure.
2385 *
2386 * @param a_pu64 Where to return the word.
2387 * @remark Implicitly references pVCpu.
2388 */
2389#ifndef IEM_WITH_SETJMP
2390# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
2391 do \
2392 { \
2393 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
2394 if (rcStrict2 != VINF_SUCCESS) \
2395 return rcStrict2; \
2396 } while (0)
2397#else
2398# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2399#endif
2400
2401
2402#ifndef IEM_WITH_SETJMP
2403/**
2404 * Fetches the next opcode byte.
2405 *
2406 * @returns Strict VBox status code.
2407 * @param pVCpu The cross context virtual CPU structure of the
2408 * calling thread.
2409 * @param pu8 Where to return the opcode byte.
2410 */
2411DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
2412{
2413 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2414 pVCpu->iem.s.offModRm = offOpcode;
2415 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2416 {
2417 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2418 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
2419 return VINF_SUCCESS;
2420 }
2421 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
2422}
2423#else /* IEM_WITH_SETJMP */
2424/**
2425 * Fetches the next opcode byte, longjmp on error.
2426 *
2427 * @returns The opcode byte.
2428 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2429 */
2430DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
2431{
2432# ifdef IEM_WITH_CODE_TLB
2433 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2434 pVCpu->iem.s.offModRm = offBuf;
2435 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2436 if (RT_LIKELY( pbBuf != NULL
2437 && offBuf < pVCpu->iem.s.cbInstrBuf))
2438 {
2439 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
2440 return pbBuf[offBuf];
2441 }
2442# else
2443 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
2444 pVCpu->iem.s.offModRm = offOpcode;
2445 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2446 {
2447 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2448 return pVCpu->iem.s.abOpcode[offOpcode];
2449 }
2450# endif
2451 return iemOpcodeGetNextU8SlowJmp(pVCpu);
2452}
2453#endif /* IEM_WITH_SETJMP */
2454
2455/**
2456 * Fetches the next opcode byte, which is a ModR/M byte, returns automatically
2457 * on failure.
2458 *
2459 * Will note down the position of the ModR/M byte for VT-x exits.
2460 *
2461 * @param a_pbRm Where to return the RM opcode byte.
2462 * @remark Implicitly references pVCpu.
2463 */
2464#ifndef IEM_WITH_SETJMP
2465# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
2466 do \
2467 { \
2468 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
2469 if (rcStrict2 == VINF_SUCCESS) \
2470 { /* likely */ } \
2471 else \
2472 return rcStrict2; \
2473 } while (0)
2474#else
2475# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
2476#endif /* IEM_WITH_SETJMP */
2477
2478
2479#ifndef IEM_WITH_SETJMP
2480
2481/**
2482 * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
2483 *
2484 * @returns Strict VBox status code.
2485 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2486 * @param pu16 Where to return the opcode word.
2487 */
2488DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16)
2489{
2490 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2491 if (rcStrict == VINF_SUCCESS)
2492 {
2493 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2494# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2495 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2496# else
2497 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2498# endif
2499 pVCpu->iem.s.offOpcode = offOpcode + 2;
2500 }
2501 else
2502 *pu16 = 0;
2503 return rcStrict;
2504}
2505
2506
2507/**
2508 * Fetches the next opcode word.
2509 *
2510 * @returns Strict VBox status code.
2511 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2512 * @param pu16 Where to return the opcode word.
2513 */
2514DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
2515{
2516 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2517 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
2518 {
2519 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
2520# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2521 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2522# else
2523 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2524# endif
2525 return VINF_SUCCESS;
2526 }
2527 return iemOpcodeGetNextU16Slow(pVCpu, pu16);
2528}
2529
2530#else /* IEM_WITH_SETJMP */
2531
2532/**
2533 * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
2534 *
2535 * @returns The opcode word.
2536 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2537 */
2538DECL_NO_INLINE(IEM_STATIC, uint16_t) iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu)
2539{
2540# ifdef IEM_WITH_CODE_TLB
2541 uint16_t u16;
2542 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
2543 return u16;
2544# else
2545 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2546 if (rcStrict == VINF_SUCCESS)
2547 {
2548 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2549 pVCpu->iem.s.offOpcode += 2;
2550# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2551 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2552# else
2553 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2554# endif
2555 }
2556 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2557# endif
2558}
2559
2560
2561/**
2562 * Fetches the next opcode word, longjmp on error.
2563 *
2564 * @returns The opcode word.
2565 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2566 */
2567DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
2568{
2569# ifdef IEM_WITH_CODE_TLB
2570 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2571 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2572 if (RT_LIKELY( pbBuf != NULL
2573 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
2574 {
2575 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
2576# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2577 return *(uint16_t const *)&pbBuf[offBuf];
2578# else
2579 return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
2580# endif
2581 }
2582# else
2583 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2584 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
2585 {
2586 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
2587# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2588 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2589# else
2590 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2591# endif
2592 }
2593# endif
2594 return iemOpcodeGetNextU16SlowJmp(pVCpu);
2595}
2596
2597#endif /* IEM_WITH_SETJMP */
2598
2599
2600/**
2601 * Fetches the next opcode word, returns automatically on failure.
2602 *
2603 * @param a_pu16 Where to return the opcode word.
2604 * @remark Implicitly references pVCpu.
2605 */
2606#ifndef IEM_WITH_SETJMP
2607# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
2608 do \
2609 { \
2610 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
2611 if (rcStrict2 != VINF_SUCCESS) \
2612 return rcStrict2; \
2613 } while (0)
2614#else
2615# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
2616#endif
2617
2618#ifndef IEM_WITH_SETJMP
2619
2620/**
2621 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
2622 *
2623 * @returns Strict VBox status code.
2624 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2625 * @param pu32 Where to return the opcode double word.
2626 */
2627DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
2628{
2629 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2630 if (rcStrict == VINF_SUCCESS)
2631 {
2632 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2633 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2634 pVCpu->iem.s.offOpcode = offOpcode + 2;
2635 }
2636 else
2637 *pu32 = 0;
2638 return rcStrict;
2639}
2640
2641
2642/**
2643 * Fetches the next opcode word, zero extending it to a double word.
2644 *
2645 * @returns Strict VBox status code.
2646 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2647 * @param pu32 Where to return the opcode double word.
2648 */
2649DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
2650{
2651 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2652 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
2653 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
2654
2655 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2656 pVCpu->iem.s.offOpcode = offOpcode + 2;
2657 return VINF_SUCCESS;
2658}
2659
2660#endif /* !IEM_WITH_SETJMP */
2661
2662
2663/**
2664 * Fetches the next opcode word and zero extends it to a double word, returns
2665 * automatically on failure.
2666 *
2667 * @param a_pu32 Where to return the opcode double word.
2668 * @remark Implicitly references pVCpu.
2669 */
2670#ifndef IEM_WITH_SETJMP
2671# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
2672 do \
2673 { \
2674 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
2675 if (rcStrict2 != VINF_SUCCESS) \
2676 return rcStrict2; \
2677 } while (0)
2678#else
2679# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
2680#endif
2681
2682#ifndef IEM_WITH_SETJMP
2683
2684/**
2685 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
2686 *
2687 * @returns Strict VBox status code.
2688 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2689 * @param pu64 Where to return the opcode quad word.
2690 */
2691DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
2692{
2693 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2694 if (rcStrict == VINF_SUCCESS)
2695 {
2696 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2697 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2698 pVCpu->iem.s.offOpcode = offOpcode + 2;
2699 }
2700 else
2701 *pu64 = 0;
2702 return rcStrict;
2703}
2704
2705
2706/**
2707 * Fetches the next opcode word, zero extending it to a quad word.
2708 *
2709 * @returns Strict VBox status code.
2710 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2711 * @param pu64 Where to return the opcode quad word.
2712 */
2713DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
2714{
2715 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2716 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
2717 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
2718
2719 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2720 pVCpu->iem.s.offOpcode = offOpcode + 2;
2721 return VINF_SUCCESS;
2722}
2723
2724#endif /* !IEM_WITH_SETJMP */
2725
2726/**
2727 * Fetches the next opcode word and zero extends it to a quad word, returns
2728 * automatically on failure.
2729 *
2730 * @param a_pu64 Where to return the opcode quad word.
2731 * @remark Implicitly references pVCpu.
2732 */
2733#ifndef IEM_WITH_SETJMP
2734# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
2735 do \
2736 { \
2737 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
2738 if (rcStrict2 != VINF_SUCCESS) \
2739 return rcStrict2; \
2740 } while (0)
2741#else
2742# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
2743#endif
2744
2745
2746#ifndef IEM_WITH_SETJMP
2747/**
2748 * Fetches the next signed word from the opcode stream.
2749 *
2750 * @returns Strict VBox status code.
2751 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2752 * @param pi16 Where to return the signed word.
2753 */
2754DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
2755{
2756 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
2757}
2758#endif /* !IEM_WITH_SETJMP */
2759
2760
2761/**
2762 * Fetches the next signed word from the opcode stream, returning automatically
2763 * on failure.
2764 *
2765 * @param a_pi16 Where to return the signed word.
2766 * @remark Implicitly references pVCpu.
2767 */
2768#ifndef IEM_WITH_SETJMP
2769# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
2770 do \
2771 { \
2772 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
2773 if (rcStrict2 != VINF_SUCCESS) \
2774 return rcStrict2; \
2775 } while (0)
2776#else
2777# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
2778#endif
2779
2780#ifndef IEM_WITH_SETJMP
2781
2782/**
2783 * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
2784 *
2785 * @returns Strict VBox status code.
2786 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2787 * @param pu32 Where to return the opcode dword.
2788 */
2789DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32)
2790{
2791 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2792 if (rcStrict == VINF_SUCCESS)
2793 {
2794 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2795# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2796 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2797# else
2798 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2799 pVCpu->iem.s.abOpcode[offOpcode + 1],
2800 pVCpu->iem.s.abOpcode[offOpcode + 2],
2801 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2802# endif
2803 pVCpu->iem.s.offOpcode = offOpcode + 4;
2804 }
2805 else
2806 *pu32 = 0;
2807 return rcStrict;
2808}
2809
2810
2811/**
2812 * Fetches the next opcode dword.
2813 *
2814 * @returns Strict VBox status code.
2815 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2816 * @param pu32 Where to return the opcode double word.
2817 */
2818DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
2819{
2820 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2821 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
2822 {
2823 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
2824# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2825 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2826# else
2827 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2828 pVCpu->iem.s.abOpcode[offOpcode + 1],
2829 pVCpu->iem.s.abOpcode[offOpcode + 2],
2830 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2831# endif
2832 return VINF_SUCCESS;
2833 }
2834 return iemOpcodeGetNextU32Slow(pVCpu, pu32);
2835}
2836
2837#else /* !IEM_WITH_SETJMP */
2838
2839/**
2840 * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
2841 *
2842 * @returns The opcode dword.
2843 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2844 */
2845DECL_NO_INLINE(IEM_STATIC, uint32_t) iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu)
2846{
2847# ifdef IEM_WITH_CODE_TLB
2848 uint32_t u32;
2849 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
2850 return u32;
2851# else
2852 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2853 if (rcStrict == VINF_SUCCESS)
2854 {
2855 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2856 pVCpu->iem.s.offOpcode = offOpcode + 4;
2857# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2858 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2859# else
2860 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2861 pVCpu->iem.s.abOpcode[offOpcode + 1],
2862 pVCpu->iem.s.abOpcode[offOpcode + 2],
2863 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2864# endif
2865 }
2866 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2867# endif
2868}
2869
2870
2871/**
2872 * Fetches the next opcode dword, longjmp on error.
2873 *
2874 * @returns The opcode dword.
2875 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2876 */
2877DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
2878{
2879# ifdef IEM_WITH_CODE_TLB
2880 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2881 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2882 if (RT_LIKELY( pbBuf != NULL
2883 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
2884 {
2885 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
2886# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2887 return *(uint32_t const *)&pbBuf[offBuf];
2888# else
2889 return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
2890 pbBuf[offBuf + 1],
2891 pbBuf[offBuf + 2],
2892 pbBuf[offBuf + 3]);
2893# endif
2894 }
2895# else
2896 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2897 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
2898 {
2899 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
2900# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2901 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2902# else
2903 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2904 pVCpu->iem.s.abOpcode[offOpcode + 1],
2905 pVCpu->iem.s.abOpcode[offOpcode + 2],
2906 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2907# endif
2908 }
2909# endif
2910 return iemOpcodeGetNextU32SlowJmp(pVCpu);
2911}
2912
2913#endif /* !IEM_WITH_SETJMP */
2914
2915
2916/**
2917 * Fetches the next opcode dword, returns automatically on failure.
2918 *
2919 * @param a_pu32 Where to return the opcode dword.
2920 * @remark Implicitly references pVCpu.
2921 */
2922#ifndef IEM_WITH_SETJMP
2923# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
2924 do \
2925 { \
2926 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
2927 if (rcStrict2 != VINF_SUCCESS) \
2928 return rcStrict2; \
2929 } while (0)
2930#else
2931# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
2932#endif
2933
2934#ifndef IEM_WITH_SETJMP
2935
2936/**
2937 * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
2938 *
2939 * @returns Strict VBox status code.
2940 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2941 * @param pu64 Where to return the opcode dword.
2942 */
2943DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
2944{
2945 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2946 if (rcStrict == VINF_SUCCESS)
2947 {
2948 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2949 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2950 pVCpu->iem.s.abOpcode[offOpcode + 1],
2951 pVCpu->iem.s.abOpcode[offOpcode + 2],
2952 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2953 pVCpu->iem.s.offOpcode = offOpcode + 4;
2954 }
2955 else
2956 *pu64 = 0;
2957 return rcStrict;
2958}
2959
2960
2961/**
2962 * Fetches the next opcode dword, zero extending it to a quad word.
2963 *
2964 * @returns Strict VBox status code.
2965 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2966 * @param pu64 Where to return the opcode quad word.
2967 */
2968DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
2969{
2970 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2971 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
2972 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
2973
2974 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2975 pVCpu->iem.s.abOpcode[offOpcode + 1],
2976 pVCpu->iem.s.abOpcode[offOpcode + 2],
2977 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2978 pVCpu->iem.s.offOpcode = offOpcode + 4;
2979 return VINF_SUCCESS;
2980}
2981
2982#endif /* !IEM_WITH_SETJMP */
2983
2984
2985/**
2986 * Fetches the next opcode dword and zero extends it to a quad word, returns
2987 * automatically on failure.
2988 *
2989 * @param a_pu64 Where to return the opcode quad word.
2990 * @remark Implicitly references pVCpu.
2991 */
2992#ifndef IEM_WITH_SETJMP
2993# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
2994 do \
2995 { \
2996 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
2997 if (rcStrict2 != VINF_SUCCESS) \
2998 return rcStrict2; \
2999 } while (0)
3000#else
3001# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
3002#endif
3003
3004
3005#ifndef IEM_WITH_SETJMP
3006/**
3007 * Fetches the next signed double word from the opcode stream.
3008 *
3009 * @returns Strict VBox status code.
3010 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3011 * @param pi32 Where to return the signed double word.
3012 */
3013DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
3014{
3015 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
3016}
3017#endif
3018
3019/**
3020 * Fetches the next signed double word from the opcode stream, returning
3021 * automatically on failure.
3022 *
3023 * @param a_pi32 Where to return the signed double word.
3024 * @remark Implicitly references pVCpu.
3025 */
3026#ifndef IEM_WITH_SETJMP
3027# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
3028 do \
3029 { \
3030 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
3031 if (rcStrict2 != VINF_SUCCESS) \
3032 return rcStrict2; \
3033 } while (0)
3034#else
3035# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
3036#endif
3037
3038#ifndef IEM_WITH_SETJMP
3039
3040/**
3041 * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
3042 *
3043 * @returns Strict VBox status code.
3044 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3045 * @param pu64 Where to return the opcode qword.
3046 */
3047DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
3048{
3049 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
3050 if (rcStrict == VINF_SUCCESS)
3051 {
3052 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3053 *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3054 pVCpu->iem.s.abOpcode[offOpcode + 1],
3055 pVCpu->iem.s.abOpcode[offOpcode + 2],
3056 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3057 pVCpu->iem.s.offOpcode = offOpcode + 4;
3058 }
3059 else
3060 *pu64 = 0;
3061 return rcStrict;
3062}
3063
3064
3065/**
3066 * Fetches the next opcode dword, sign extending it into a quad word.
3067 *
3068 * @returns Strict VBox status code.
3069 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3070 * @param pu64 Where to return the opcode quad word.
3071 */
3072DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
3073{
3074 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
3075 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
3076 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
3077
3078 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3079 pVCpu->iem.s.abOpcode[offOpcode + 1],
3080 pVCpu->iem.s.abOpcode[offOpcode + 2],
3081 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3082 *pu64 = i32;
3083 pVCpu->iem.s.offOpcode = offOpcode + 4;
3084 return VINF_SUCCESS;
3085}
3086
3087#endif /* !IEM_WITH_SETJMP */
3088
3089
3090/**
3091 * Fetches the next opcode double word and sign extends it to a quad word,
3092 * returns automatically on failure.
3093 *
3094 * @param a_pu64 Where to return the opcode quad word.
3095 * @remark Implicitly references pVCpu.
3096 */
3097#ifndef IEM_WITH_SETJMP
3098# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
3099 do \
3100 { \
3101 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
3102 if (rcStrict2 != VINF_SUCCESS) \
3103 return rcStrict2; \
3104 } while (0)
3105#else
3106# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
3107#endif
3108
3109#ifndef IEM_WITH_SETJMP
3110
3111/**
3112 * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
3113 *
3114 * @returns Strict VBox status code.
3115 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3116 * @param pu64 Where to return the opcode qword.
3117 */
3118DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64)
3119{
3120 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
3121 if (rcStrict == VINF_SUCCESS)
3122 {
3123 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3124# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3125 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3126# else
3127 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3128 pVCpu->iem.s.abOpcode[offOpcode + 1],
3129 pVCpu->iem.s.abOpcode[offOpcode + 2],
3130 pVCpu->iem.s.abOpcode[offOpcode + 3],
3131 pVCpu->iem.s.abOpcode[offOpcode + 4],
3132 pVCpu->iem.s.abOpcode[offOpcode + 5],
3133 pVCpu->iem.s.abOpcode[offOpcode + 6],
3134 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3135# endif
3136 pVCpu->iem.s.offOpcode = offOpcode + 8;
3137 }
3138 else
3139 *pu64 = 0;
3140 return rcStrict;
3141}
3142
3143
3144/**
3145 * Fetches the next opcode qword.
3146 *
3147 * @returns Strict VBox status code.
3148 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3149 * @param pu64 Where to return the opcode qword.
3150 */
3151DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
3152{
3153 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
3154 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
3155 {
3156# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3157 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3158# else
3159 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3160 pVCpu->iem.s.abOpcode[offOpcode + 1],
3161 pVCpu->iem.s.abOpcode[offOpcode + 2],
3162 pVCpu->iem.s.abOpcode[offOpcode + 3],
3163 pVCpu->iem.s.abOpcode[offOpcode + 4],
3164 pVCpu->iem.s.abOpcode[offOpcode + 5],
3165 pVCpu->iem.s.abOpcode[offOpcode + 6],
3166 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3167# endif
3168 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
3169 return VINF_SUCCESS;
3170 }
3171 return iemOpcodeGetNextU64Slow(pVCpu, pu64);
3172}
3173
3174#else /* IEM_WITH_SETJMP */
3175
3176/**
3177 * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
3178 *
3179 * @returns The opcode qword.
3180 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3181 */
3182DECL_NO_INLINE(IEM_STATIC, uint64_t) iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu)
3183{
3184# ifdef IEM_WITH_CODE_TLB
3185 uint64_t u64;
3186 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
3187 return u64;
3188# else
3189 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
3190 if (rcStrict == VINF_SUCCESS)
3191 {
3192 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3193 pVCpu->iem.s.offOpcode = offOpcode + 8;
3194# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3195 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3196# else
3197 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3198 pVCpu->iem.s.abOpcode[offOpcode + 1],
3199 pVCpu->iem.s.abOpcode[offOpcode + 2],
3200 pVCpu->iem.s.abOpcode[offOpcode + 3],
3201 pVCpu->iem.s.abOpcode[offOpcode + 4],
3202 pVCpu->iem.s.abOpcode[offOpcode + 5],
3203 pVCpu->iem.s.abOpcode[offOpcode + 6],
3204 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3205# endif
3206 }
3207 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3208# endif
3209}
3210
3211
3212/**
3213 * Fetches the next opcode qword, longjmp on error.
3214 *
3215 * @returns The opcode qword.
3216 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3217 */
3218DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
3219{
3220# ifdef IEM_WITH_CODE_TLB
3221 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
3222 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
3223 if (RT_LIKELY( pbBuf != NULL
3224 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
3225 {
3226 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
3227# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3228 return *(uint64_t const *)&pbBuf[offBuf];
3229# else
3230 return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
3231 pbBuf[offBuf + 1],
3232 pbBuf[offBuf + 2],
3233 pbBuf[offBuf + 3],
3234 pbBuf[offBuf + 4],
3235 pbBuf[offBuf + 5],
3236 pbBuf[offBuf + 6],
3237 pbBuf[offBuf + 7]);
3238# endif
3239 }
3240# else
3241 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
3242 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
3243 {
3244 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
3245# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3246 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3247# else
3248 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3249 pVCpu->iem.s.abOpcode[offOpcode + 1],
3250 pVCpu->iem.s.abOpcode[offOpcode + 2],
3251 pVCpu->iem.s.abOpcode[offOpcode + 3],
3252 pVCpu->iem.s.abOpcode[offOpcode + 4],
3253 pVCpu->iem.s.abOpcode[offOpcode + 5],
3254 pVCpu->iem.s.abOpcode[offOpcode + 6],
3255 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3256# endif
3257 }
3258# endif
3259 return iemOpcodeGetNextU64SlowJmp(pVCpu);
3260}
3261
3262#endif /* IEM_WITH_SETJMP */
3263
3264/**
3265 * Fetches the next opcode quad word, returns automatically on failure.
3266 *
3267 * @param a_pu64 Where to return the opcode quad word.
3268 * @remark Implicitly references pVCpu.
3269 */
3270#ifndef IEM_WITH_SETJMP
3271# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
3272 do \
3273 { \
3274 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
3275 if (rcStrict2 != VINF_SUCCESS) \
3276 return rcStrict2; \
3277 } while (0)
3278#else
3279# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
3280#endif
3281
3282
3283/** @name Misc Worker Functions.
3284 * @{
3285 */
3286
3287/**
3288 * Gets the exception class for the specified exception vector.
3289 *
3290 * @returns The class of the specified exception.
3291 * @param uVector The exception vector.
3292 */
3293IEM_STATIC IEMXCPTCLASS iemGetXcptClass(uint8_t uVector)
3294{
3295 Assert(uVector <= X86_XCPT_LAST);
3296 switch (uVector)
3297 {
3298 case X86_XCPT_DE:
3299 case X86_XCPT_TS:
3300 case X86_XCPT_NP:
3301 case X86_XCPT_SS:
3302 case X86_XCPT_GP:
3303 case X86_XCPT_SX: /* AMD only */
3304 return IEMXCPTCLASS_CONTRIBUTORY;
3305
3306 case X86_XCPT_PF:
3307 case X86_XCPT_VE: /* Intel only */
3308 return IEMXCPTCLASS_PAGE_FAULT;
3309
3310 case X86_XCPT_DF:
3311 return IEMXCPTCLASS_DOUBLE_FAULT;
3312 }
3313 return IEMXCPTCLASS_BENIGN;
3314}
3315
3316
3317/**
3318 * Evaluates how to handle an exception caused during delivery of another event
3319 * (exception / interrupt).
3320 *
3321 * @returns How to handle the recursive exception.
3322 * @param pVCpu The cross context virtual CPU structure of the
3323 * calling thread.
3324 * @param fPrevFlags The flags of the previous event.
3325 * @param uPrevVector The vector of the previous event.
3326 * @param fCurFlags The flags of the current exception.
3327 * @param uCurVector The vector of the current exception.
3328 * @param pfXcptRaiseInfo Where to store additional information about the
3329 * exception condition. Optional.
3330 */
3331VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
3332 uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
3333{
3334 /*
3335 * Only CPU exceptions can be raised while delivering other events, software interrupt
3336 * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
3337 */
3338 AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
3339 Assert(pVCpu); RT_NOREF(pVCpu);
3340 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
3341
3342 IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
3343 IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
3344 if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3345 {
3346 IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
3347 if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
3348 {
3349 IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
3350 if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
3351 && ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
3352 || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
3353 {
3354 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
3355 fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
3356 : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
3357 Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
3358 uCurVector, pVCpu->cpum.GstCtx.cr2));
3359 }
3360 else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
3361 && enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
3362 {
3363 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
3364 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
3365 }
3366 else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
3367 && ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
3368 || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
3369 {
3370 enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
3371 Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
3372 }
3373 }
3374 else
3375 {
3376 if (uPrevVector == X86_XCPT_NMI)
3377 {
3378 fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
3379 if (uCurVector == X86_XCPT_PF)
3380 {
3381 fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF;
3382 Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
3383 }
3384 }
3385 else if ( uPrevVector == X86_XCPT_AC
3386 && uCurVector == X86_XCPT_AC)
3387 {
3388 enmRaise = IEMXCPTRAISE_CPU_HANG;
3389 fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
3390 Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
3391 }
3392 }
3393 }
3394 else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
3395 {
3396 fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
3397 if (uCurVector == X86_XCPT_PF)
3398 fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF;
3399 }
3400 else
3401 {
3402 Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
3403 fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
3404 }
3405
3406 if (pfXcptRaiseInfo)
3407 *pfXcptRaiseInfo = fRaiseInfo;
3408 return enmRaise;
3409}
3410
3411
3412/**
3413 * Enters the CPU shutdown state initiated by a triple fault or other
3414 * unrecoverable conditions.
3415 *
3416 * @returns Strict VBox status code.
3417 * @param pVCpu The cross context virtual CPU structure of the
3418 * calling thread.
3419 */
3420IEM_STATIC VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu)
3421{
3422 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3423 IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
3424
3425 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
3426 {
3427 Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
3428 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
3429 }
3430
3431 RT_NOREF(pVCpu);
3432 return VINF_EM_TRIPLE_FAULT;
3433}
3434
3435
3436/**
3437 * Validates a new SS segment.
3438 *
3439 * @returns VBox strict status code.
3440 * @param pVCpu The cross context virtual CPU structure of the
3441 * calling thread.
3442 * @param NewSS The new SS selctor.
3443 * @param uCpl The CPL to load the stack for.
3444 * @param pDesc Where to return the descriptor.
3445 */
3446IEM_STATIC VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
3447{
3448 /* Null selectors are not allowed (we're not called for dispatching
3449 interrupts with SS=0 in long mode). */
3450 if (!(NewSS & X86_SEL_MASK_OFF_RPL))
3451 {
3452 Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
3453 return iemRaiseTaskSwitchFault0(pVCpu);
3454 }
3455
3456 /** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
3457 if ((NewSS & X86_SEL_RPL) != uCpl)
3458 {
3459 Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
3460 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3461 }
3462
3463 /*
3464 * Read the descriptor.
3465 */
3466 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
3467 if (rcStrict != VINF_SUCCESS)
3468 return rcStrict;
3469
3470 /*
3471 * Perform the descriptor validation documented for LSS, POP SS and MOV SS.
3472 */
3473 if (!pDesc->Legacy.Gen.u1DescType)
3474 {
3475 Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
3476 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3477 }
3478
3479 if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
3480 || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
3481 {
3482 Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
3483 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3484 }
3485 if (pDesc->Legacy.Gen.u2Dpl != uCpl)
3486 {
3487 Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
3488 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3489 }
3490
3491 /* Is it there? */
3492 /** @todo testcase: Is this checked before the canonical / limit check below? */
3493 if (!pDesc->Legacy.Gen.u1Present)
3494 {
3495 Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
3496 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
3497 }
3498
3499 return VINF_SUCCESS;
3500}
3501
3502
3503/**
3504 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
3505 * not (kind of obsolete now).
3506 *
3507 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
3508 */
3509#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
3510
3511/**
3512 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
3513 *
3514 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
3515 * @param a_fEfl The new EFLAGS.
3516 */
3517#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
3518
3519/** @} */
3520
3521
3522/** @name Raising Exceptions.
3523 *
3524 * @{
3525 */
3526
3527
3528/**
3529 * Loads the specified stack far pointer from the TSS.
3530 *
3531 * @returns VBox strict status code.
3532 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3533 * @param uCpl The CPL to load the stack for.
3534 * @param pSelSS Where to return the new stack segment.
3535 * @param puEsp Where to return the new stack pointer.
3536 */
3537IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp)
3538{
3539 VBOXSTRICTRC rcStrict;
3540 Assert(uCpl < 4);
3541
3542 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
3543 switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
3544 {
3545 /*
3546 * 16-bit TSS (X86TSS16).
3547 */
3548 case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
3549 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
3550 {
3551 uint32_t off = uCpl * 4 + 2;
3552 if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
3553 {
3554 /** @todo check actual access pattern here. */
3555 uint32_t u32Tmp = 0; /* gcc maybe... */
3556 rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3557 if (rcStrict == VINF_SUCCESS)
3558 {
3559 *puEsp = RT_LOWORD(u32Tmp);
3560 *pSelSS = RT_HIWORD(u32Tmp);
3561 return VINF_SUCCESS;
3562 }
3563 }
3564 else
3565 {
3566 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
3567 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3568 }
3569 break;
3570 }
3571
3572 /*
3573 * 32-bit TSS (X86TSS32).
3574 */
3575 case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
3576 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
3577 {
3578 uint32_t off = uCpl * 8 + 4;
3579 if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
3580 {
3581/** @todo check actual access pattern here. */
3582 uint64_t u64Tmp;
3583 rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3584 if (rcStrict == VINF_SUCCESS)
3585 {
3586 *puEsp = u64Tmp & UINT32_MAX;
3587 *pSelSS = (RTSEL)(u64Tmp >> 32);
3588 return VINF_SUCCESS;
3589 }
3590 }
3591 else
3592 {
3593 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
3594 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3595 }
3596 break;
3597 }
3598
3599 default:
3600 AssertFailed();
3601 rcStrict = VERR_IEM_IPE_4;
3602 break;
3603 }
3604
3605 *puEsp = 0; /* make gcc happy */
3606 *pSelSS = 0; /* make gcc happy */
3607 return rcStrict;
3608}
3609
3610
3611/**
3612 * Loads the specified stack pointer from the 64-bit TSS.
3613 *
3614 * @returns VBox strict status code.
3615 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3616 * @param uCpl The CPL to load the stack for.
3617 * @param uIst The interrupt stack table index, 0 if to use uCpl.
3618 * @param puRsp Where to return the new stack pointer.
3619 */
3620IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp)
3621{
3622 Assert(uCpl < 4);
3623 Assert(uIst < 8);
3624 *puRsp = 0; /* make gcc happy */
3625
3626 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
3627 AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
3628
3629 uint32_t off;
3630 if (uIst)
3631 off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
3632 else
3633 off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
3634 if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
3635 {
3636 Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
3637 return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3638 }
3639
3640 return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3641}
3642
3643
3644/**
3645 * Adjust the CPU state according to the exception being raised.
3646 *
3647 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3648 * @param u8Vector The exception that has been raised.
3649 */
3650DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
3651{
3652 switch (u8Vector)
3653 {
3654 case X86_XCPT_DB:
3655 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
3656 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3657 break;
3658 /** @todo Read the AMD and Intel exception reference... */
3659 }
3660}
3661
3662
3663/**
3664 * Implements exceptions and interrupts for real mode.
3665 *
3666 * @returns VBox strict status code.
3667 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3668 * @param cbInstr The number of bytes to offset rIP by in the return
3669 * address.
3670 * @param u8Vector The interrupt / exception vector number.
3671 * @param fFlags The flags.
3672 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3673 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3674 */
3675IEM_STATIC VBOXSTRICTRC
3676iemRaiseXcptOrIntInRealMode(PVMCPUCC pVCpu,
3677 uint8_t cbInstr,
3678 uint8_t u8Vector,
3679 uint32_t fFlags,
3680 uint16_t uErr,
3681 uint64_t uCr2)
3682{
3683 NOREF(uErr); NOREF(uCr2);
3684 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3685
3686 /*
3687 * Read the IDT entry.
3688 */
3689 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
3690 {
3691 Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3692 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3693 }
3694 RTFAR16 Idte;
3695 VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector);
3696 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3697 {
3698 Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3699 return rcStrict;
3700 }
3701
3702 /*
3703 * Push the stack frame.
3704 */
3705 uint16_t *pu16Frame;
3706 uint64_t uNewRsp;
3707 rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, (void **)&pu16Frame, &uNewRsp);
3708 if (rcStrict != VINF_SUCCESS)
3709 return rcStrict;
3710
3711 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3712#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
3713 AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
3714 if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
3715 fEfl |= UINT16_C(0xf000);
3716#endif
3717 pu16Frame[2] = (uint16_t)fEfl;
3718 pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
3719 pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3720 rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
3721 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3722 return rcStrict;
3723
3724 /*
3725 * Load the vector address into cs:ip and make exception specific state
3726 * adjustments.
3727 */
3728 pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
3729 pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
3730 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3731 pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
3732 /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
3733 pVCpu->cpum.GstCtx.rip = Idte.off;
3734 fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC);
3735 IEMMISC_SET_EFL(pVCpu, fEfl);
3736
3737 /** @todo do we actually do this in real mode? */
3738 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3739 iemRaiseXcptAdjustState(pVCpu, u8Vector);
3740
3741 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3742}
3743
3744
3745/**
3746 * Loads a NULL data selector into when coming from V8086 mode.
3747 *
3748 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3749 * @param pSReg Pointer to the segment register.
3750 */
3751IEM_STATIC void iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
3752{
3753 pSReg->Sel = 0;
3754 pSReg->ValidSel = 0;
3755 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3756 {
3757 /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
3758 pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D;
3759 pSReg->Attr.u |= X86DESCATTR_UNUSABLE;
3760 }
3761 else
3762 {
3763 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3764 /** @todo check this on AMD-V */
3765 pSReg->u64Base = 0;
3766 pSReg->u32Limit = 0;
3767 }
3768}
3769
3770
3771/**
3772 * Loads a segment selector during a task switch in V8086 mode.
3773 *
3774 * @param pSReg Pointer to the segment register.
3775 * @param uSel The selector value to load.
3776 */
3777IEM_STATIC void iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
3778{
3779 /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3780 pSReg->Sel = uSel;
3781 pSReg->ValidSel = uSel;
3782 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3783 pSReg->u64Base = uSel << 4;
3784 pSReg->u32Limit = 0xffff;
3785 pSReg->Attr.u = 0xf3;
3786}
3787
3788
3789/**
3790 * Loads a NULL data selector into a selector register, both the hidden and
3791 * visible parts, in protected mode.
3792 *
3793 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3794 * @param pSReg Pointer to the segment register.
3795 * @param uRpl The RPL.
3796 */
3797IEM_STATIC void iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
3798{
3799 /** @todo Testcase: write a testcase checking what happends when loading a NULL
3800 * data selector in protected mode. */
3801 pSReg->Sel = uRpl;
3802 pSReg->ValidSel = uRpl;
3803 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3804 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3805 {
3806 /* VT-x (Intel 3960x) observed doing something like this. */
3807 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
3808 pSReg->u32Limit = UINT32_MAX;
3809 pSReg->u64Base = 0;
3810 }
3811 else
3812 {
3813 pSReg->Attr.u = X86DESCATTR_UNUSABLE;
3814 pSReg->u32Limit = 0;
3815 pSReg->u64Base = 0;
3816 }
3817}
3818
3819
3820/**
3821 * Loads a segment selector during a task switch in protected mode.
3822 *
3823 * In this task switch scenario, we would throw \#TS exceptions rather than
3824 * \#GPs.
3825 *
3826 * @returns VBox strict status code.
3827 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3828 * @param pSReg Pointer to the segment register.
3829 * @param uSel The new selector value.
3830 *
3831 * @remarks This does _not_ handle CS or SS.
3832 * @remarks This expects pVCpu->iem.s.uCpl to be up to date.
3833 */
3834IEM_STATIC VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel)
3835{
3836 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
3837
3838 /* Null data selector. */
3839 if (!(uSel & X86_SEL_MASK_OFF_RPL))
3840 {
3841 iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
3842 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
3843 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
3844 return VINF_SUCCESS;
3845 }
3846
3847 /* Fetch the descriptor. */
3848 IEMSELDESC Desc;
3849 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
3850 if (rcStrict != VINF_SUCCESS)
3851 {
3852 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
3853 VBOXSTRICTRC_VAL(rcStrict)));
3854 return rcStrict;
3855 }
3856
3857 /* Must be a data segment or readable code segment. */
3858 if ( !Desc.Legacy.Gen.u1DescType
3859 || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3860 {
3861 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
3862 Desc.Legacy.Gen.u4Type));
3863 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3864 }
3865
3866 /* Check privileges for data segments and non-conforming code segments. */
3867 if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
3868 != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
3869 {
3870 /* The RPL and the new CPL must be less than or equal to the DPL. */
3871 if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
3872 || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
3873 {
3874 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
3875 uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3876 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3877 }
3878 }
3879
3880 /* Is it there? */
3881 if (!Desc.Legacy.Gen.u1Present)
3882 {
3883 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
3884 return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3885 }
3886
3887 /* The base and limit. */
3888 uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
3889 uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
3890
3891 /*
3892 * Ok, everything checked out fine. Now set the accessed bit before
3893 * committing the result into the registers.
3894 */
3895 if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3896 {
3897 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
3898 if (rcStrict != VINF_SUCCESS)
3899 return rcStrict;
3900 Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3901 }
3902
3903 /* Commit */
3904 pSReg->Sel = uSel;
3905 pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
3906 pSReg->u32Limit = cbLimit;
3907 pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
3908 pSReg->ValidSel = uSel;
3909 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3910 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3911 pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
3912
3913 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
3914 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
3915 return VINF_SUCCESS;
3916}
3917
3918
3919/**
3920 * Performs a task switch.
3921 *
3922 * If the task switch is the result of a JMP, CALL or IRET instruction, the
3923 * caller is responsible for performing the necessary checks (like DPL, TSS
3924 * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
3925 * reference for JMP, CALL, IRET.
3926 *
3927 * If the task switch is the due to a software interrupt or hardware exception,
3928 * the caller is responsible for validating the TSS selector and descriptor. See
3929 * Intel Instruction reference for INT n.
3930 *
3931 * @returns VBox strict status code.
3932 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3933 * @param enmTaskSwitch The cause of the task switch.
3934 * @param uNextEip The EIP effective after the task switch.
3935 * @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
3936 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3937 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3938 * @param SelTSS The TSS selector of the new task.
3939 * @param pNewDescTSS Pointer to the new TSS descriptor.
3940 */
3941IEM_STATIC VBOXSTRICTRC
3942iemTaskSwitch(PVMCPUCC pVCpu,
3943 IEMTASKSWITCH enmTaskSwitch,
3944 uint32_t uNextEip,
3945 uint32_t fFlags,
3946 uint16_t uErr,
3947 uint64_t uCr2,
3948 RTSEL SelTSS,
3949 PIEMSELDESC pNewDescTSS)
3950{
3951 Assert(!IEM_IS_REAL_MODE(pVCpu));
3952 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
3953 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3954
3955 uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
3956 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
3957 || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
3958 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
3959 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
3960
3961 bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
3962 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
3963
3964 Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
3965 fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
3966
3967 /* Update CR2 in case it's a page-fault. */
3968 /** @todo This should probably be done much earlier in IEM/PGM. See
3969 * @bugref{5653#c49}. */
3970 if (fFlags & IEM_XCPT_FLAGS_CR2)
3971 pVCpu->cpum.GstCtx.cr2 = uCr2;
3972
3973 /*
3974 * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
3975 * subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
3976 */
3977 uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
3978 uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
3979 if (uNewTSSLimit < uNewTSSLimitMin)
3980 {
3981 Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
3982 enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
3983 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
3984 }
3985
3986 /*
3987 * Task switches in VMX non-root mode always cause task switches.
3988 * The new TSS must have been read and validated (DPL, limits etc.) before a
3989 * task-switch VM-exit commences.
3990 *
3991 * See Intel spec. 25.4.2 "Treatment of Task Switches".
3992 */
3993 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
3994 {
3995 Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
3996 IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
3997 }
3998
3999 /*
4000 * The SVM nested-guest intercept for task-switch takes priority over all exceptions
4001 * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
4002 */
4003 if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
4004 {
4005 uint32_t const uExitInfo1 = SelTSS;
4006 uint32_t uExitInfo2 = uErr;
4007 switch (enmTaskSwitch)
4008 {
4009 case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break;
4010 case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break;
4011 default: break;
4012 }
4013 if (fFlags & IEM_XCPT_FLAGS_ERR)
4014 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
4015 if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
4016 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
4017
4018 Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
4019 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
4020 RT_NOREF2(uExitInfo1, uExitInfo2);
4021 }
4022
4023 /*
4024 * Check the current TSS limit. The last written byte to the current TSS during the
4025 * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
4026 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
4027 *
4028 * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
4029 * end up with smaller than "legal" TSS limits.
4030 */
4031 uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
4032 uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
4033 if (uCurTSSLimit < uCurTSSLimitMin)
4034 {
4035 Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
4036 enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
4037 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
4038 }
4039
4040 /*
4041 * Verify that the new TSS can be accessed and map it. Map only the required contents
4042 * and not the entire TSS.
4043 */
4044 void *pvNewTSS;
4045 uint32_t const cbNewTSS = uNewTSSLimitMin + 1;
4046 RTGCPTR const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
4047 AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
4048 /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
4049 * not perform correct translation if this happens. See Intel spec. 7.2.1
4050 * "Task-State Segment". */
4051 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW);
4052 if (rcStrict != VINF_SUCCESS)
4053 {
4054 Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
4055 cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
4056 return rcStrict;
4057 }
4058
4059 /*
4060 * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
4061 */
4062 uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
4063 if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
4064 || enmTaskSwitch == IEMTASKSWITCH_IRET)
4065 {
4066 PX86DESC pDescCurTSS;
4067 rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX,
4068 pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
4069 if (rcStrict != VINF_SUCCESS)
4070 {
4071 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4072 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4073 return rcStrict;
4074 }
4075
4076 pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4077 rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
4078 if (rcStrict != VINF_SUCCESS)
4079 {
4080 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4081 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4082 return rcStrict;
4083 }
4084
4085 /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
4086 if (enmTaskSwitch == IEMTASKSWITCH_IRET)
4087 {
4088 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
4089 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
4090 u32EFlags &= ~X86_EFL_NT;
4091 }
4092 }
4093
4094 /*
4095 * Save the CPU state into the current TSS.
4096 */
4097 RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
4098 if (GCPtrNewTSS == GCPtrCurTSS)
4099 {
4100 Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
4101 Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
4102 pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax,
4103 pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
4104 pVCpu->cpum.GstCtx.ldtr.Sel));
4105 }
4106 if (fIsNewTSS386)
4107 {
4108 /*
4109 * Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
4110 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
4111 */
4112 void *pvCurTSS32;
4113 uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
4114 uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
4115 AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
4116 rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
4117 if (rcStrict != VINF_SUCCESS)
4118 {
4119 Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
4120 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
4121 return rcStrict;
4122 }
4123
4124 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
4125 PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
4126 pCurTSS32->eip = uNextEip;
4127 pCurTSS32->eflags = u32EFlags;
4128 pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
4129 pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
4130 pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
4131 pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
4132 pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
4133 pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
4134 pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
4135 pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
4136 pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
4137 pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
4138 pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
4139 pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
4140 pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
4141 pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
4142
4143 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
4144 if (rcStrict != VINF_SUCCESS)
4145 {
4146 Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
4147 VBOXSTRICTRC_VAL(rcStrict)));
4148 return rcStrict;
4149 }
4150 }
4151 else
4152 {
4153 /*
4154 * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
4155 */
4156 void *pvCurTSS16;
4157 uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
4158 uint32_t const cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
4159 AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
4160 rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
4161 if (rcStrict != VINF_SUCCESS)
4162 {
4163 Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
4164 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
4165 return rcStrict;
4166 }
4167
4168 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
4169 PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
4170 pCurTSS16->ip = uNextEip;
4171 pCurTSS16->flags = u32EFlags;
4172 pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
4173 pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
4174 pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
4175 pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
4176 pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
4177 pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
4178 pCurTSS16->si = pVCpu->cpum.GstCtx.si;
4179 pCurTSS16->di = pVCpu->cpum.GstCtx.di;
4180 pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
4181 pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
4182 pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
4183 pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
4184
4185 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
4186 if (rcStrict != VINF_SUCCESS)
4187 {
4188 Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
4189 VBOXSTRICTRC_VAL(rcStrict)));
4190 return rcStrict;
4191 }
4192 }
4193
4194 /*
4195 * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
4196 */
4197 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
4198 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
4199 {
4200 /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
4201 PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
4202 pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
4203 }
4204
4205 /*
4206 * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
4207 * it's done further below with error handling (e.g. CR3 changes will go through PGM).
4208 */
4209 uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
4210 uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
4211 bool fNewDebugTrap;
4212 if (fIsNewTSS386)
4213 {
4214 PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
4215 uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
4216 uNewEip = pNewTSS32->eip;
4217 uNewEflags = pNewTSS32->eflags;
4218 uNewEax = pNewTSS32->eax;
4219 uNewEcx = pNewTSS32->ecx;
4220 uNewEdx = pNewTSS32->edx;
4221 uNewEbx = pNewTSS32->ebx;
4222 uNewEsp = pNewTSS32->esp;
4223 uNewEbp = pNewTSS32->ebp;
4224 uNewEsi = pNewTSS32->esi;
4225 uNewEdi = pNewTSS32->edi;
4226 uNewES = pNewTSS32->es;
4227 uNewCS = pNewTSS32->cs;
4228 uNewSS = pNewTSS32->ss;
4229 uNewDS = pNewTSS32->ds;
4230 uNewFS = pNewTSS32->fs;
4231 uNewGS = pNewTSS32->gs;
4232 uNewLdt = pNewTSS32->selLdt;
4233 fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
4234 }
4235 else
4236 {
4237 PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
4238 uNewCr3 = 0;
4239 uNewEip = pNewTSS16->ip;
4240 uNewEflags = pNewTSS16->flags;
4241 uNewEax = UINT32_C(0xffff0000) | pNewTSS16->ax;
4242 uNewEcx = UINT32_C(0xffff0000) | pNewTSS16->cx;
4243 uNewEdx = UINT32_C(0xffff0000) | pNewTSS16->dx;
4244 uNewEbx = UINT32_C(0xffff0000) | pNewTSS16->bx;
4245 uNewEsp = UINT32_C(0xffff0000) | pNewTSS16->sp;
4246 uNewEbp = UINT32_C(0xffff0000) | pNewTSS16->bp;
4247 uNewEsi = UINT32_C(0xffff0000) | pNewTSS16->si;
4248 uNewEdi = UINT32_C(0xffff0000) | pNewTSS16->di;
4249 uNewES = pNewTSS16->es;
4250 uNewCS = pNewTSS16->cs;
4251 uNewSS = pNewTSS16->ss;
4252 uNewDS = pNewTSS16->ds;
4253 uNewFS = 0;
4254 uNewGS = 0;
4255 uNewLdt = pNewTSS16->selLdt;
4256 fNewDebugTrap = false;
4257 }
4258
4259 if (GCPtrNewTSS == GCPtrCurTSS)
4260 Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
4261 uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
4262
4263 /*
4264 * We're done accessing the new TSS.
4265 */
4266 rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
4267 if (rcStrict != VINF_SUCCESS)
4268 {
4269 Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
4270 return rcStrict;
4271 }
4272
4273 /*
4274 * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
4275 */
4276 if (enmTaskSwitch != IEMTASKSWITCH_IRET)
4277 {
4278 rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX,
4279 pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
4280 if (rcStrict != VINF_SUCCESS)
4281 {
4282 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4283 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4284 return rcStrict;
4285 }
4286
4287 /* Check that the descriptor indicates the new TSS is available (not busy). */
4288 AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
4289 || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
4290 ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
4291
4292 pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4293 rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
4294 if (rcStrict != VINF_SUCCESS)
4295 {
4296 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4297 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4298 return rcStrict;
4299 }
4300 }
4301
4302 /*
4303 * From this point on, we're technically in the new task. We will defer exceptions
4304 * until the completion of the task switch but before executing any instructions in the new task.
4305 */
4306 pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
4307 pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
4308 pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
4309 pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
4310 pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
4311 pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
4312 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
4313
4314 /* Set the busy bit in TR. */
4315 pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4316
4317 /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
4318 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
4319 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
4320 {
4321 uNewEflags |= X86_EFL_NT;
4322 }
4323
4324 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
4325 pVCpu->cpum.GstCtx.cr0 |= X86_CR0_TS;
4326 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
4327
4328 pVCpu->cpum.GstCtx.eip = uNewEip;
4329 pVCpu->cpum.GstCtx.eax = uNewEax;
4330 pVCpu->cpum.GstCtx.ecx = uNewEcx;
4331 pVCpu->cpum.GstCtx.edx = uNewEdx;
4332 pVCpu->cpum.GstCtx.ebx = uNewEbx;
4333 pVCpu->cpum.GstCtx.esp = uNewEsp;
4334 pVCpu->cpum.GstCtx.ebp = uNewEbp;
4335 pVCpu->cpum.GstCtx.esi = uNewEsi;
4336 pVCpu->cpum.GstCtx.edi = uNewEdi;
4337
4338 uNewEflags &= X86_EFL_LIVE_MASK;
4339 uNewEflags |= X86_EFL_RA1_MASK;
4340 IEMMISC_SET_EFL(pVCpu, uNewEflags);
4341
4342 /*
4343 * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
4344 * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
4345 * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
4346 */
4347 pVCpu->cpum.GstCtx.es.Sel = uNewES;
4348 pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
4349
4350 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
4351 pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
4352
4353 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
4354 pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
4355
4356 pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
4357 pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
4358
4359 pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
4360 pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
4361
4362 pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
4363 pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
4364 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
4365
4366 pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
4367 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
4368 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
4369 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
4370
4371 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4372 {
4373 pVCpu->cpum.GstCtx.es.Attr.u |= X86DESCATTR_UNUSABLE;
4374 pVCpu->cpum.GstCtx.cs.Attr.u |= X86DESCATTR_UNUSABLE;
4375 pVCpu->cpum.GstCtx.ss.Attr.u |= X86DESCATTR_UNUSABLE;
4376 pVCpu->cpum.GstCtx.ds.Attr.u |= X86DESCATTR_UNUSABLE;
4377 pVCpu->cpum.GstCtx.fs.Attr.u |= X86DESCATTR_UNUSABLE;
4378 pVCpu->cpum.GstCtx.gs.Attr.u |= X86DESCATTR_UNUSABLE;
4379 pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE;
4380 }
4381
4382 /*
4383 * Switch CR3 for the new task.
4384 */
4385 if ( fIsNewTSS386
4386 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
4387 {
4388 /** @todo Should we update and flush TLBs only if CR3 value actually changes? */
4389 int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
4390 AssertRCSuccessReturn(rc, rc);
4391
4392 /* Inform PGM. */
4393 /** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
4394 rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
4395 AssertRCReturn(rc, rc);
4396 /* ignore informational status codes */
4397
4398 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
4399 }
4400
4401 /*
4402 * Switch LDTR for the new task.
4403 */
4404 if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
4405 iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
4406 else
4407 {
4408 Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
4409
4410 IEMSELDESC DescNewLdt;
4411 rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
4412 if (rcStrict != VINF_SUCCESS)
4413 {
4414 Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
4415 uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
4416 return rcStrict;
4417 }
4418 if ( !DescNewLdt.Legacy.Gen.u1Present
4419 || DescNewLdt.Legacy.Gen.u1DescType
4420 || DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
4421 {
4422 Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
4423 uNewLdt, DescNewLdt.Legacy.u));
4424 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
4425 }
4426
4427 pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
4428 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
4429 pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
4430 pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
4431 pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
4432 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4433 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
4434 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
4435 }
4436
4437 IEMSELDESC DescSS;
4438 if (IEM_IS_V86_MODE(pVCpu))
4439 {
4440 pVCpu->iem.s.uCpl = 3;
4441 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
4442 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
4443 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
4444 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
4445 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
4446 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
4447
4448 /* Quick fix: fake DescSS. */ /** @todo fix the code further down? */
4449 DescSS.Legacy.u = 0;
4450 DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
4451 DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
4452 DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
4453 DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
4454 DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
4455 DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
4456 DescSS.Legacy.Gen.u2Dpl = 3;
4457 }
4458 else
4459 {
4460 uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
4461
4462 /*
4463 * Load the stack segment for the new task.
4464 */
4465 if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
4466 {
4467 Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
4468 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4469 }
4470
4471 /* Fetch the descriptor. */
4472 rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
4473 if (rcStrict != VINF_SUCCESS)
4474 {
4475 Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
4476 VBOXSTRICTRC_VAL(rcStrict)));
4477 return rcStrict;
4478 }
4479
4480 /* SS must be a data segment and writable. */
4481 if ( !DescSS.Legacy.Gen.u1DescType
4482 || (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
4483 || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
4484 {
4485 Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
4486 uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
4487 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4488 }
4489
4490 /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
4491 if ( (uNewSS & X86_SEL_RPL) != uNewCpl
4492 || DescSS.Legacy.Gen.u2Dpl != uNewCpl)
4493 {
4494 Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
4495 uNewCpl));
4496 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4497 }
4498
4499 /* Is it there? */
4500 if (!DescSS.Legacy.Gen.u1Present)
4501 {
4502 Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
4503 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4504 }
4505
4506 uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
4507 uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
4508
4509 /* Set the accessed bit before committing the result into SS. */
4510 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4511 {
4512 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
4513 if (rcStrict != VINF_SUCCESS)
4514 return rcStrict;
4515 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
4516 }
4517
4518 /* Commit SS. */
4519 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
4520 pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
4521 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
4522 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
4523 pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
4524 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4525 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
4526
4527 /* CPL has changed, update IEM before loading rest of segments. */
4528 pVCpu->iem.s.uCpl = uNewCpl;
4529
4530 /*
4531 * Load the data segments for the new task.
4532 */
4533 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
4534 if (rcStrict != VINF_SUCCESS)
4535 return rcStrict;
4536 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
4537 if (rcStrict != VINF_SUCCESS)
4538 return rcStrict;
4539 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
4540 if (rcStrict != VINF_SUCCESS)
4541 return rcStrict;
4542 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
4543 if (rcStrict != VINF_SUCCESS)
4544 return rcStrict;
4545
4546 /*
4547 * Load the code segment for the new task.
4548 */
4549 if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
4550 {
4551 Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
4552 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4553 }
4554
4555 /* Fetch the descriptor. */
4556 IEMSELDESC DescCS;
4557 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
4558 if (rcStrict != VINF_SUCCESS)
4559 {
4560 Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
4561 return rcStrict;
4562 }
4563
4564 /* CS must be a code segment. */
4565 if ( !DescCS.Legacy.Gen.u1DescType
4566 || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
4567 {
4568 Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
4569 DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
4570 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4571 }
4572
4573 /* For conforming CS, DPL must be less than or equal to the RPL. */
4574 if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
4575 && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
4576 {
4577 Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
4578 DescCS.Legacy.Gen.u2Dpl));
4579 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4580 }
4581
4582 /* For non-conforming CS, DPL must match RPL. */
4583 if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
4584 && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
4585 {
4586 Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
4587 DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
4588 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4589 }
4590
4591 /* Is it there? */
4592 if (!DescCS.Legacy.Gen.u1Present)
4593 {
4594 Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
4595 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4596 }
4597
4598 cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
4599 u64Base = X86DESC_BASE(&DescCS.Legacy);
4600
4601 /* Set the accessed bit before committing the result into CS. */
4602 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4603 {
4604 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
4605 if (rcStrict != VINF_SUCCESS)
4606 return rcStrict;
4607 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
4608 }
4609
4610 /* Commit CS. */
4611 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
4612 pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
4613 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
4614 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
4615 pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
4616 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4617 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
4618 }
4619
4620 /** @todo Debug trap. */
4621 if (fIsNewTSS386 && fNewDebugTrap)
4622 Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
4623
4624 /*
4625 * Construct the error code masks based on what caused this task switch.
4626 * See Intel Instruction reference for INT.
4627 */
4628 uint16_t uExt;
4629 if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
4630 && ( !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
4631 || (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
4632 {
4633 uExt = 1;
4634 }
4635 else
4636 uExt = 0;
4637
4638 /*
4639 * Push any error code on to the new stack.
4640 */
4641 if (fFlags & IEM_XCPT_FLAGS_ERR)
4642 {
4643 Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
4644 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
4645 uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
4646
4647 /* Check that there is sufficient space on the stack. */
4648 /** @todo Factor out segment limit checking for normal/expand down segments
4649 * into a separate function. */
4650 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
4651 {
4652 if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
4653 || pVCpu->cpum.GstCtx.esp < cbStackFrame)
4654 {
4655 /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
4656 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
4657 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
4658 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
4659 }
4660 }
4661 else
4662 {
4663 if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
4664 || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
4665 {
4666 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
4667 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
4668 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
4669 }
4670 }
4671
4672
4673 if (fIsNewTSS386)
4674 rcStrict = iemMemStackPushU32(pVCpu, uErr);
4675 else
4676 rcStrict = iemMemStackPushU16(pVCpu, uErr);
4677 if (rcStrict != VINF_SUCCESS)
4678 {
4679 Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
4680 fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
4681 return rcStrict;
4682 }
4683 }
4684
4685 /* Check the new EIP against the new CS limit. */
4686 if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
4687 {
4688 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
4689 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
4690 /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
4691 return iemRaiseGeneralProtectionFault(pVCpu, uExt);
4692 }
4693
4694 Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
4695 pVCpu->cpum.GstCtx.ss.Sel));
4696 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
4697}
4698
4699
4700/**
4701 * Implements exceptions and interrupts for protected mode.
4702 *
4703 * @returns VBox strict status code.
4704 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4705 * @param cbInstr The number of bytes to offset rIP by in the return
4706 * address.
4707 * @param u8Vector The interrupt / exception vector number.
4708 * @param fFlags The flags.
4709 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
4710 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
4711 */
4712IEM_STATIC VBOXSTRICTRC
4713iemRaiseXcptOrIntInProtMode(PVMCPUCC pVCpu,
4714 uint8_t cbInstr,
4715 uint8_t u8Vector,
4716 uint32_t fFlags,
4717 uint16_t uErr,
4718 uint64_t uCr2)
4719{
4720 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
4721
4722 /*
4723 * Read the IDT entry.
4724 */
4725 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
4726 {
4727 Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
4728 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4729 }
4730 X86DESC Idte;
4731 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
4732 pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
4733 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
4734 {
4735 Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
4736 return rcStrict;
4737 }
4738 Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
4739 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
4740 Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
4741
4742 /*
4743 * Check the descriptor type, DPL and such.
4744 * ASSUMES this is done in the same order as described for call-gate calls.
4745 */
4746 if (Idte.Gate.u1DescType)
4747 {
4748 Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
4749 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4750 }
4751 bool fTaskGate = false;
4752 uint8_t f32BitGate = true;
4753 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
4754 switch (Idte.Gate.u4Type)
4755 {
4756 case X86_SEL_TYPE_SYS_UNDEFINED:
4757 case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
4758 case X86_SEL_TYPE_SYS_LDT:
4759 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
4760 case X86_SEL_TYPE_SYS_286_CALL_GATE:
4761 case X86_SEL_TYPE_SYS_UNDEFINED2:
4762 case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
4763 case X86_SEL_TYPE_SYS_UNDEFINED3:
4764 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
4765 case X86_SEL_TYPE_SYS_386_CALL_GATE:
4766 case X86_SEL_TYPE_SYS_UNDEFINED4:
4767 {
4768 /** @todo check what actually happens when the type is wrong...
4769 * esp. call gates. */
4770 Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
4771 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4772 }
4773
4774 case X86_SEL_TYPE_SYS_286_INT_GATE:
4775 f32BitGate = false;
4776 RT_FALL_THRU();
4777 case X86_SEL_TYPE_SYS_386_INT_GATE:
4778 fEflToClear |= X86_EFL_IF;
4779 break;
4780
4781 case X86_SEL_TYPE_SYS_TASK_GATE:
4782 fTaskGate = true;
4783#ifndef IEM_IMPLEMENTS_TASKSWITCH
4784 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
4785#endif
4786 break;
4787
4788 case X86_SEL_TYPE_SYS_286_TRAP_GATE:
4789 f32BitGate = false;
4790 case X86_SEL_TYPE_SYS_386_TRAP_GATE:
4791 break;
4792
4793 IEM_NOT_REACHED_DEFAULT_CASE_RET();
4794 }
4795
4796 /* Check DPL against CPL if applicable. */
4797 if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
4798 {
4799 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
4800 {
4801 Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
4802 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4803 }
4804 }
4805
4806 /* Is it there? */
4807 if (!Idte.Gate.u1Present)
4808 {
4809 Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
4810 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4811 }
4812
4813 /* Is it a task-gate? */
4814 if (fTaskGate)
4815 {
4816 /*
4817 * Construct the error code masks based on what caused this task switch.
4818 * See Intel Instruction reference for INT.
4819 */
4820 uint16_t const uExt = ( (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
4821 && !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
4822 uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
4823 RTSEL SelTSS = Idte.Gate.u16Sel;
4824
4825 /*
4826 * Fetch the TSS descriptor in the GDT.
4827 */
4828 IEMSELDESC DescTSS;
4829 rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt);
4830 if (rcStrict != VINF_SUCCESS)
4831 {
4832 Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
4833 VBOXSTRICTRC_VAL(rcStrict)));
4834 return rcStrict;
4835 }
4836
4837 /* The TSS descriptor must be a system segment and be available (not busy). */
4838 if ( DescTSS.Legacy.Gen.u1DescType
4839 || ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
4840 && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
4841 {
4842 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
4843 u8Vector, SelTSS, DescTSS.Legacy.au64));
4844 return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt);
4845 }
4846
4847 /* The TSS must be present. */
4848 if (!DescTSS.Legacy.Gen.u1Present)
4849 {
4850 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
4851 return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt);
4852 }
4853
4854 /* Do the actual task switch. */
4855 return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
4856 (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
4857 fFlags, uErr, uCr2, SelTSS, &DescTSS);
4858 }
4859
4860 /* A null CS is bad. */
4861 RTSEL NewCS = Idte.Gate.u16Sel;
4862 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
4863 {
4864 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
4865 return iemRaiseGeneralProtectionFault0(pVCpu);
4866 }
4867
4868 /* Fetch the descriptor for the new CS. */
4869 IEMSELDESC DescCS;
4870 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
4871 if (rcStrict != VINF_SUCCESS)
4872 {
4873 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
4874 return rcStrict;
4875 }
4876
4877 /* Must be a code segment. */
4878 if (!DescCS.Legacy.Gen.u1DescType)
4879 {
4880 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
4881 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4882 }
4883 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
4884 {
4885 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
4886 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4887 }
4888
4889 /* Don't allow lowering the privilege level. */
4890 /** @todo Does the lowering of privileges apply to software interrupts
4891 * only? This has bearings on the more-privileged or
4892 * same-privilege stack behavior further down. A testcase would
4893 * be nice. */
4894 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
4895 {
4896 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
4897 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
4898 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4899 }
4900
4901 /* Make sure the selector is present. */
4902 if (!DescCS.Legacy.Gen.u1Present)
4903 {
4904 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
4905 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
4906 }
4907
4908 /* Check the new EIP against the new CS limit. */
4909 uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
4910 || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
4911 ? Idte.Gate.u16OffsetLow
4912 : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
4913 uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
4914 if (uNewEip > cbLimitCS)
4915 {
4916 Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
4917 u8Vector, uNewEip, cbLimitCS, NewCS));
4918 return iemRaiseGeneralProtectionFault(pVCpu, 0);
4919 }
4920 Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
4921
4922 /* Calc the flag image to push. */
4923 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
4924 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
4925 fEfl &= ~X86_EFL_RF;
4926 else
4927 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
4928
4929 /* From V8086 mode only go to CPL 0. */
4930 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
4931 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
4932 if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
4933 {
4934 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
4935 return iemRaiseGeneralProtectionFault(pVCpu, 0);
4936 }
4937
4938 /*
4939 * If the privilege level changes, we need to get a new stack from the TSS.
4940 * This in turns means validating the new SS and ESP...
4941 */
4942 if (uNewCpl != pVCpu->iem.s.uCpl)
4943 {
4944 RTSEL NewSS;
4945 uint32_t uNewEsp;
4946 rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
4947 if (rcStrict != VINF_SUCCESS)
4948 return rcStrict;
4949
4950 IEMSELDESC DescSS;
4951 rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
4952 if (rcStrict != VINF_SUCCESS)
4953 return rcStrict;
4954 /* If the new SS is 16-bit, we are only going to use SP, not ESP. */
4955 if (!DescSS.Legacy.Gen.u1DefBig)
4956 {
4957 Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
4958 uNewEsp = (uint16_t)uNewEsp;
4959 }
4960
4961 Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
4962
4963 /* Check that there is sufficient space for the stack frame. */
4964 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
4965 uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
4966 ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
4967 : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
4968
4969 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
4970 {
4971 if ( uNewEsp - 1 > cbLimitSS
4972 || uNewEsp < cbStackFrame)
4973 {
4974 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
4975 u8Vector, NewSS, uNewEsp, cbStackFrame));
4976 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
4977 }
4978 }
4979 else
4980 {
4981 if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
4982 || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
4983 {
4984 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
4985 u8Vector, NewSS, uNewEsp, cbStackFrame));
4986 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
4987 }
4988 }
4989
4990 /*
4991 * Start making changes.
4992 */
4993
4994 /* Set the new CPL so that stack accesses use it. */
4995 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
4996 pVCpu->iem.s.uCpl = uNewCpl;
4997
4998 /* Create the stack frame. */
4999 RTPTRUNION uStackFrame;
5000 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
5001 uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy), IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
5002 if (rcStrict != VINF_SUCCESS)
5003 return rcStrict;
5004 void * const pvStackFrame = uStackFrame.pv;
5005 if (f32BitGate)
5006 {
5007 if (fFlags & IEM_XCPT_FLAGS_ERR)
5008 *uStackFrame.pu32++ = uErr;
5009 uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5010 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
5011 uStackFrame.pu32[2] = fEfl;
5012 uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
5013 uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
5014 Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
5015 if (fEfl & X86_EFL_VM)
5016 {
5017 uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
5018 uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
5019 uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
5020 uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
5021 uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
5022 }
5023 }
5024 else
5025 {
5026 if (fFlags & IEM_XCPT_FLAGS_ERR)
5027 *uStackFrame.pu16++ = uErr;
5028 uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
5029 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
5030 uStackFrame.pu16[2] = fEfl;
5031 uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
5032 uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
5033 Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
5034 if (fEfl & X86_EFL_VM)
5035 {
5036 uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
5037 uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
5038 uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
5039 uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
5040 uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
5041 }
5042 }
5043 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
5044 if (rcStrict != VINF_SUCCESS)
5045 return rcStrict;
5046
5047 /* Mark the selectors 'accessed' (hope this is the correct time). */
5048 /** @todo testcase: excatly _when_ are the accessed bits set - before or
5049 * after pushing the stack frame? (Write protect the gdt + stack to
5050 * find out.) */
5051 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5052 {
5053 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5054 if (rcStrict != VINF_SUCCESS)
5055 return rcStrict;
5056 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5057 }
5058
5059 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5060 {
5061 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
5062 if (rcStrict != VINF_SUCCESS)
5063 return rcStrict;
5064 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5065 }
5066
5067 /*
5068 * Start comitting the register changes (joins with the DPL=CPL branch).
5069 */
5070 pVCpu->cpum.GstCtx.ss.Sel = NewSS;
5071 pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
5072 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
5073 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
5074 pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
5075 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
5076 /** @todo When coming from 32-bit code and operating with a 16-bit TSS and
5077 * 16-bit handler, the high word of ESP remains unchanged (i.e. only
5078 * SP is loaded).
5079 * Need to check the other combinations too:
5080 * - 16-bit TSS, 32-bit handler
5081 * - 32-bit TSS, 16-bit handler */
5082 if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
5083 pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
5084 else
5085 pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
5086
5087 if (fEfl & X86_EFL_VM)
5088 {
5089 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
5090 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
5091 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
5092 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
5093 }
5094 }
5095 /*
5096 * Same privilege, no stack change and smaller stack frame.
5097 */
5098 else
5099 {
5100 uint64_t uNewRsp;
5101 RTPTRUNION uStackFrame;
5102 uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
5103 rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
5104 if (rcStrict != VINF_SUCCESS)
5105 return rcStrict;
5106 void * const pvStackFrame = uStackFrame.pv;
5107
5108 if (f32BitGate)
5109 {
5110 if (fFlags & IEM_XCPT_FLAGS_ERR)
5111 *uStackFrame.pu32++ = uErr;
5112 uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5113 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
5114 uStackFrame.pu32[2] = fEfl;
5115 }
5116 else
5117 {
5118 if (fFlags & IEM_XCPT_FLAGS_ERR)
5119 *uStackFrame.pu16++ = uErr;
5120 uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5121 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
5122 uStackFrame.pu16[2] = fEfl;
5123 }
5124 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
5125 if (rcStrict != VINF_SUCCESS)
5126 return rcStrict;
5127
5128 /* Mark the CS selector as 'accessed'. */
5129 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5130 {
5131 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5132 if (rcStrict != VINF_SUCCESS)
5133 return rcStrict;
5134 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5135 }
5136
5137 /*
5138 * Start committing the register changes (joins with the other branch).
5139 */
5140 pVCpu->cpum.GstCtx.rsp = uNewRsp;
5141 }
5142
5143 /* ... register committing continues. */
5144 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5145 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5146 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
5147 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
5148 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
5149 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
5150
5151 pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
5152 fEfl &= ~fEflToClear;
5153 IEMMISC_SET_EFL(pVCpu, fEfl);
5154
5155 if (fFlags & IEM_XCPT_FLAGS_CR2)
5156 pVCpu->cpum.GstCtx.cr2 = uCr2;
5157
5158 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
5159 iemRaiseXcptAdjustState(pVCpu, u8Vector);
5160
5161 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
5162}
5163
5164
5165/**
5166 * Implements exceptions and interrupts for long mode.
5167 *
5168 * @returns VBox strict status code.
5169 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5170 * @param cbInstr The number of bytes to offset rIP by in the return
5171 * address.
5172 * @param u8Vector The interrupt / exception vector number.
5173 * @param fFlags The flags.
5174 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
5175 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
5176 */
5177IEM_STATIC VBOXSTRICTRC
5178iemRaiseXcptOrIntInLongMode(PVMCPUCC pVCpu,
5179 uint8_t cbInstr,
5180 uint8_t u8Vector,
5181 uint32_t fFlags,
5182 uint16_t uErr,
5183 uint64_t uCr2)
5184{
5185 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5186
5187 /*
5188 * Read the IDT entry.
5189 */
5190 uint16_t offIdt = (uint16_t)u8Vector << 4;
5191 if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
5192 {
5193 Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
5194 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5195 }
5196 X86DESC64 Idte;
5197#ifdef _MSC_VER /* Shut up silly compiler warning. */
5198 Idte.au64[0] = 0;
5199 Idte.au64[1] = 0;
5200#endif
5201 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
5202 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5203 rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
5204 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
5205 {
5206 Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
5207 return rcStrict;
5208 }
5209 Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
5210 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
5211 Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
5212
5213 /*
5214 * Check the descriptor type, DPL and such.
5215 * ASSUMES this is done in the same order as described for call-gate calls.
5216 */
5217 if (Idte.Gate.u1DescType)
5218 {
5219 Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
5220 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5221 }
5222 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
5223 switch (Idte.Gate.u4Type)
5224 {
5225 case AMD64_SEL_TYPE_SYS_INT_GATE:
5226 fEflToClear |= X86_EFL_IF;
5227 break;
5228 case AMD64_SEL_TYPE_SYS_TRAP_GATE:
5229 break;
5230
5231 default:
5232 Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
5233 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5234 }
5235
5236 /* Check DPL against CPL if applicable. */
5237 if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
5238 {
5239 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
5240 {
5241 Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
5242 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5243 }
5244 }
5245
5246 /* Is it there? */
5247 if (!Idte.Gate.u1Present)
5248 {
5249 Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
5250 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5251 }
5252
5253 /* A null CS is bad. */
5254 RTSEL NewCS = Idte.Gate.u16Sel;
5255 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
5256 {
5257 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
5258 return iemRaiseGeneralProtectionFault0(pVCpu);
5259 }
5260
5261 /* Fetch the descriptor for the new CS. */
5262 IEMSELDESC DescCS;
5263 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
5264 if (rcStrict != VINF_SUCCESS)
5265 {
5266 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
5267 return rcStrict;
5268 }
5269
5270 /* Must be a 64-bit code segment. */
5271 if (!DescCS.Long.Gen.u1DescType)
5272 {
5273 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
5274 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5275 }
5276 if ( !DescCS.Long.Gen.u1Long
5277 || DescCS.Long.Gen.u1DefBig
5278 || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
5279 {
5280 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
5281 u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
5282 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5283 }
5284
5285 /* Don't allow lowering the privilege level. For non-conforming CS
5286 selectors, the CS.DPL sets the privilege level the trap/interrupt
5287 handler runs at. For conforming CS selectors, the CPL remains
5288 unchanged, but the CS.DPL must be <= CPL. */
5289 /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
5290 * when CPU in Ring-0. Result \#GP? */
5291 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
5292 {
5293 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
5294 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
5295 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5296 }
5297
5298
5299 /* Make sure the selector is present. */
5300 if (!DescCS.Legacy.Gen.u1Present)
5301 {
5302 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
5303 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
5304 }
5305
5306 /* Check that the new RIP is canonical. */
5307 uint64_t const uNewRip = Idte.Gate.u16OffsetLow
5308 | ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
5309 | ((uint64_t)Idte.Gate.u32OffsetTop << 32);
5310 if (!IEM_IS_CANONICAL(uNewRip))
5311 {
5312 Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
5313 return iemRaiseGeneralProtectionFault0(pVCpu);
5314 }
5315
5316 /*
5317 * If the privilege level changes or if the IST isn't zero, we need to get
5318 * a new stack from the TSS.
5319 */
5320 uint64_t uNewRsp;
5321 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
5322 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
5323 if ( uNewCpl != pVCpu->iem.s.uCpl
5324 || Idte.Gate.u3IST != 0)
5325 {
5326 rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
5327 if (rcStrict != VINF_SUCCESS)
5328 return rcStrict;
5329 }
5330 else
5331 uNewRsp = pVCpu->cpum.GstCtx.rsp;
5332 uNewRsp &= ~(uint64_t)0xf;
5333
5334 /*
5335 * Calc the flag image to push.
5336 */
5337 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
5338 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
5339 fEfl &= ~X86_EFL_RF;
5340 else
5341 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
5342
5343 /*
5344 * Start making changes.
5345 */
5346 /* Set the new CPL so that stack accesses use it. */
5347 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
5348 pVCpu->iem.s.uCpl = uNewCpl;
5349
5350 /* Create the stack frame. */
5351 uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
5352 RTPTRUNION uStackFrame;
5353 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
5354 uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
5355 if (rcStrict != VINF_SUCCESS)
5356 return rcStrict;
5357 void * const pvStackFrame = uStackFrame.pv;
5358
5359 if (fFlags & IEM_XCPT_FLAGS_ERR)
5360 *uStackFrame.pu64++ = uErr;
5361 uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
5362 uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */
5363 uStackFrame.pu64[2] = fEfl;
5364 uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
5365 uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
5366 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
5367 if (rcStrict != VINF_SUCCESS)
5368 return rcStrict;
5369
5370 /* Mark the CS selectors 'accessed' (hope this is the correct time). */
5371 /** @todo testcase: excatly _when_ are the accessed bits set - before or
5372 * after pushing the stack frame? (Write protect the gdt + stack to
5373 * find out.) */
5374 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5375 {
5376 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5377 if (rcStrict != VINF_SUCCESS)
5378 return rcStrict;
5379 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5380 }
5381
5382 /*
5383 * Start comitting the register changes.
5384 */
5385 /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
5386 * hidden registers when interrupting 32-bit or 16-bit code! */
5387 if (uNewCpl != uOldCpl)
5388 {
5389 pVCpu->cpum.GstCtx.ss.Sel = 0 | uNewCpl;
5390 pVCpu->cpum.GstCtx.ss.ValidSel = 0 | uNewCpl;
5391 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
5392 pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
5393 pVCpu->cpum.GstCtx.ss.u64Base = 0;
5394 pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE;
5395 }
5396 pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
5397 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5398 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5399 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
5400 pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
5401 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
5402 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
5403 pVCpu->cpum.GstCtx.rip = uNewRip;
5404
5405 fEfl &= ~fEflToClear;
5406 IEMMISC_SET_EFL(pVCpu, fEfl);
5407
5408 if (fFlags & IEM_XCPT_FLAGS_CR2)
5409 pVCpu->cpum.GstCtx.cr2 = uCr2;
5410
5411 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
5412 iemRaiseXcptAdjustState(pVCpu, u8Vector);
5413
5414 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
5415}
5416
5417
5418/**
5419 * Implements exceptions and interrupts.
5420 *
5421 * All exceptions and interrupts goes thru this function!
5422 *
5423 * @returns VBox strict status code.
5424 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5425 * @param cbInstr The number of bytes to offset rIP by in the return
5426 * address.
5427 * @param u8Vector The interrupt / exception vector number.
5428 * @param fFlags The flags.
5429 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
5430 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
5431 */
5432DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC)
5433iemRaiseXcptOrInt(PVMCPUCC pVCpu,
5434 uint8_t cbInstr,
5435 uint8_t u8Vector,
5436 uint32_t fFlags,
5437 uint16_t uErr,
5438 uint64_t uCr2)
5439{
5440 /*
5441 * Get all the state that we might need here.
5442 */
5443 IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5444 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5445
5446#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
5447 /*
5448 * Flush prefetch buffer
5449 */
5450 pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
5451#endif
5452
5453 /*
5454 * Perform the V8086 IOPL check and upgrade the fault without nesting.
5455 */
5456 if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
5457 && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
5458 && (fFlags & ( IEM_XCPT_FLAGS_T_SOFT_INT
5459 | IEM_XCPT_FLAGS_BP_INSTR
5460 | IEM_XCPT_FLAGS_ICEBP_INSTR
5461 | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
5462 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
5463 {
5464 Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
5465 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
5466 u8Vector = X86_XCPT_GP;
5467 uErr = 0;
5468 }
5469#ifdef DBGFTRACE_ENABLED
5470 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
5471 pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
5472 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
5473#endif
5474
5475 /*
5476 * Evaluate whether NMI blocking should be in effect.
5477 * Normally, NMI blocking is in effect whenever we inject an NMI.
5478 */
5479 bool fBlockNmi;
5480 if ( u8Vector == X86_XCPT_NMI
5481 && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
5482 fBlockNmi = true;
5483 else
5484 fBlockNmi = false;
5485
5486#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5487 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5488 {
5489 VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
5490 if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5491 return rcStrict0;
5492
5493 /* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
5494 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
5495 {
5496 Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
5497 fBlockNmi = false;
5498 }
5499 }
5500#endif
5501
5502#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5503 if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
5504 {
5505 /*
5506 * If the event is being injected as part of VMRUN, it isn't subject to event
5507 * intercepts in the nested-guest. However, secondary exceptions that occur
5508 * during injection of any event -are- subject to exception intercepts.
5509 *
5510 * See AMD spec. 15.20 "Event Injection".
5511 */
5512 if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
5513 pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
5514 else
5515 {
5516 /*
5517 * Check and handle if the event being raised is intercepted.
5518 */
5519 VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
5520 if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
5521 return rcStrict0;
5522 }
5523 }
5524#endif
5525
5526 /*
5527 * Set NMI blocking if necessary.
5528 */
5529 if ( fBlockNmi
5530 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
5531 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5532
5533 /*
5534 * Do recursion accounting.
5535 */
5536 uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
5537 uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
5538 if (pVCpu->iem.s.cXcptRecursions == 0)
5539 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
5540 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
5541 else
5542 {
5543 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
5544 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
5545 pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
5546
5547 if (pVCpu->iem.s.cXcptRecursions >= 4)
5548 {
5549#ifdef DEBUG_bird
5550 AssertFailed();
5551#endif
5552 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
5553 }
5554
5555 /*
5556 * Evaluate the sequence of recurring events.
5557 */
5558 IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
5559 NULL /* pXcptRaiseInfo */);
5560 if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
5561 { /* likely */ }
5562 else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
5563 {
5564 Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
5565 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
5566 u8Vector = X86_XCPT_DF;
5567 uErr = 0;
5568#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
5569 /* VMX nested-guest #DF intercept needs to be checked here. */
5570 if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
5571 {
5572 VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
5573 if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
5574 return rcStrict0;
5575 }
5576#endif
5577 /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
5578 if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
5579 IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
5580 }
5581 else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
5582 {
5583 Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
5584 return iemInitiateCpuShutdown(pVCpu);
5585 }
5586 else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
5587 {
5588 /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
5589 Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
5590 if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
5591 && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
5592 return VERR_EM_GUEST_CPU_HANG;
5593 }
5594 else
5595 {
5596 AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
5597 enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
5598 return VERR_IEM_IPE_9;
5599 }
5600
5601 /*
5602 * The 'EXT' bit is set when an exception occurs during deliver of an external
5603 * event (such as an interrupt or earlier exception)[1]. Privileged software
5604 * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
5605 * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
5606 *
5607 * [1] - Intel spec. 6.13 "Error Code"
5608 * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
5609 * [3] - Intel Instruction reference for INT n.
5610 */
5611 if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR))
5612 && (fFlags & IEM_XCPT_FLAGS_ERR)
5613 && u8Vector != X86_XCPT_PF
5614 && u8Vector != X86_XCPT_DF)
5615 {
5616 uErr |= X86_TRAP_ERR_EXTERNAL;
5617 }
5618 }
5619
5620 pVCpu->iem.s.cXcptRecursions++;
5621 pVCpu->iem.s.uCurXcpt = u8Vector;
5622 pVCpu->iem.s.fCurXcpt = fFlags;
5623 pVCpu->iem.s.uCurXcptErr = uErr;
5624 pVCpu->iem.s.uCurXcptCr2 = uCr2;
5625
5626 /*
5627 * Extensive logging.
5628 */
5629#if defined(LOG_ENABLED) && defined(IN_RING3)
5630 if (LogIs3Enabled())
5631 {
5632 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
5633 PVM pVM = pVCpu->CTX_SUFF(pVM);
5634 char szRegs[4096];
5635 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
5636 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
5637 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
5638 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
5639 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
5640 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
5641 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
5642 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
5643 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
5644 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
5645 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
5646 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
5647 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
5648 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
5649 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
5650 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
5651 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
5652 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
5653 " efer=%016VR{efer}\n"
5654 " pat=%016VR{pat}\n"
5655 " sf_mask=%016VR{sf_mask}\n"
5656 "krnl_gs_base=%016VR{krnl_gs_base}\n"
5657 " lstar=%016VR{lstar}\n"
5658 " star=%016VR{star} cstar=%016VR{cstar}\n"
5659 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
5660 );
5661
5662 char szInstr[256];
5663 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
5664 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
5665 szInstr, sizeof(szInstr), NULL);
5666 Log3(("%s%s\n", szRegs, szInstr));
5667 }
5668#endif /* LOG_ENABLED */
5669
5670 /*
5671 * Call the mode specific worker function.
5672 */
5673 VBOXSTRICTRC rcStrict;
5674 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
5675 rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5676 else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
5677 rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5678 else
5679 rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5680
5681 /* Flush the prefetch buffer. */
5682#ifdef IEM_WITH_CODE_TLB
5683 pVCpu->iem.s.pbInstrBuf = NULL;
5684#else
5685 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
5686#endif
5687
5688 /*
5689 * Unwind.
5690 */
5691 pVCpu->iem.s.cXcptRecursions--;
5692 pVCpu->iem.s.uCurXcpt = uPrevXcpt;
5693 pVCpu->iem.s.fCurXcpt = fPrevXcpt;
5694 Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
5695 VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
5696 pVCpu->iem.s.cXcptRecursions + 1));
5697 return rcStrict;
5698}
5699
5700#ifdef IEM_WITH_SETJMP
5701/**
5702 * See iemRaiseXcptOrInt. Will not return.
5703 */
5704IEM_STATIC DECL_NO_RETURN(void)
5705iemRaiseXcptOrIntJmp(PVMCPUCC pVCpu,
5706 uint8_t cbInstr,
5707 uint8_t u8Vector,
5708 uint32_t fFlags,
5709 uint16_t uErr,
5710 uint64_t uCr2)
5711{
5712 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5713 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
5714}
5715#endif
5716
5717
5718/** \#DE - 00. */
5719DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDivideError(PVMCPUCC pVCpu)
5720{
5721 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5722}
5723
5724
5725/** \#DB - 01.
5726 * @note This automatically clear DR7.GD. */
5727DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDebugException(PVMCPUCC pVCpu)
5728{
5729 /** @todo set/clear RF. */
5730 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
5731 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5732}
5733
5734
5735/** \#BR - 05. */
5736DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu)
5737{
5738 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5739}
5740
5741
5742/** \#UD - 06. */
5743DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PVMCPUCC pVCpu)
5744{
5745 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5746}
5747
5748
5749/** \#NM - 07. */
5750DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu)
5751{
5752 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5753}
5754
5755
5756/** \#TS(err) - 0a. */
5757DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr)
5758{
5759 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5760}
5761
5762
5763/** \#TS(tr) - 0a. */
5764DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu)
5765{
5766 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5767 pVCpu->cpum.GstCtx.tr.Sel, 0);
5768}
5769
5770
5771/** \#TS(0) - 0a. */
5772DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu)
5773{
5774 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5775 0, 0);
5776}
5777
5778
5779/** \#TS(err) - 0a. */
5780DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel)
5781{
5782 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5783 uSel & X86_SEL_MASK_OFF_RPL, 0);
5784}
5785
5786
5787/** \#NP(err) - 0b. */
5788DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr)
5789{
5790 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5791}
5792
5793
5794/** \#NP(sel) - 0b. */
5795DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel)
5796{
5797 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5798 uSel & ~X86_SEL_RPL, 0);
5799}
5800
5801
5802/** \#SS(seg) - 0c. */
5803DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel)
5804{
5805 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5806 uSel & ~X86_SEL_RPL, 0);
5807}
5808
5809
5810/** \#SS(err) - 0c. */
5811DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr)
5812{
5813 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5814}
5815
5816
5817/** \#GP(n) - 0d. */
5818DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr)
5819{
5820 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5821}
5822
5823
5824/** \#GP(0) - 0d. */
5825DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu)
5826{
5827 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5828}
5829
5830#ifdef IEM_WITH_SETJMP
5831/** \#GP(0) - 0d. */
5832DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu)
5833{
5834 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5835}
5836#endif
5837
5838
5839/** \#GP(sel) - 0d. */
5840DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel)
5841{
5842 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5843 Sel & ~X86_SEL_RPL, 0);
5844}
5845
5846
5847/** \#GP(0) - 0d. */
5848DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseNotCanonical(PVMCPUCC pVCpu)
5849{
5850 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5851}
5852
5853
5854/** \#GP(sel) - 0d. */
5855DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
5856{
5857 NOREF(iSegReg); NOREF(fAccess);
5858 return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
5859 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5860}
5861
5862#ifdef IEM_WITH_SETJMP
5863/** \#GP(sel) - 0d, longjmp. */
5864DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
5865{
5866 NOREF(iSegReg); NOREF(fAccess);
5867 iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
5868 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5869}
5870#endif
5871
5872/** \#GP(sel) - 0d. */
5873DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel)
5874{
5875 NOREF(Sel);
5876 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5877}
5878
5879#ifdef IEM_WITH_SETJMP
5880/** \#GP(sel) - 0d, longjmp. */
5881DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel)
5882{
5883 NOREF(Sel);
5884 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5885}
5886#endif
5887
5888
5889/** \#GP(sel) - 0d. */
5890DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess)
5891{
5892 NOREF(iSegReg); NOREF(fAccess);
5893 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5894}
5895
5896#ifdef IEM_WITH_SETJMP
5897/** \#GP(sel) - 0d, longjmp. */
5898DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg,
5899 uint32_t fAccess)
5900{
5901 NOREF(iSegReg); NOREF(fAccess);
5902 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5903}
5904#endif
5905
5906
5907/** \#PF(n) - 0e. */
5908DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
5909{
5910 uint16_t uErr;
5911 switch (rc)
5912 {
5913 case VERR_PAGE_NOT_PRESENT:
5914 case VERR_PAGE_TABLE_NOT_PRESENT:
5915 case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
5916 case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
5917 uErr = 0;
5918 break;
5919
5920 default:
5921 AssertMsgFailed(("%Rrc\n", rc));
5922 RT_FALL_THRU();
5923 case VERR_ACCESS_DENIED:
5924 uErr = X86_TRAP_PF_P;
5925 break;
5926
5927 /** @todo reserved */
5928 }
5929
5930 if (pVCpu->iem.s.uCpl == 3)
5931 uErr |= X86_TRAP_PF_US;
5932
5933 if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
5934 && ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
5935 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
5936 uErr |= X86_TRAP_PF_ID;
5937
5938#if 0 /* This is so much non-sense, really. Why was it done like that? */
5939 /* Note! RW access callers reporting a WRITE protection fault, will clear
5940 the READ flag before calling. So, read-modify-write accesses (RW)
5941 can safely be reported as READ faults. */
5942 if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
5943 uErr |= X86_TRAP_PF_RW;
5944#else
5945 if (fAccess & IEM_ACCESS_TYPE_WRITE)
5946 {
5947 /// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
5948 /// (regardless of outcome of the comparison in the latter case).
5949 //if (!(fAccess & IEM_ACCESS_TYPE_READ))
5950 uErr |= X86_TRAP_PF_RW;
5951 }
5952#endif
5953
5954 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2,
5955 uErr, GCPtrWhere);
5956}
5957
5958#ifdef IEM_WITH_SETJMP
5959/** \#PF(n) - 0e, longjmp. */
5960IEM_STATIC DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
5961{
5962 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
5963}
5964#endif
5965
5966
5967/** \#MF(0) - 10. */
5968DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseMathFault(PVMCPUCC pVCpu)
5969{
5970 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5971}
5972
5973
5974/** \#AC(0) - 11. */
5975DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PVMCPUCC pVCpu)
5976{
5977 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5978}
5979
5980
5981/**
5982 * Macro for calling iemCImplRaiseDivideError().
5983 *
5984 * This enables us to add/remove arguments and force different levels of
5985 * inlining as we wish.
5986 *
5987 * @return Strict VBox status code.
5988 */
5989#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
5990IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
5991{
5992 NOREF(cbInstr);
5993 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5994}
5995
5996
5997/**
5998 * Macro for calling iemCImplRaiseInvalidLockPrefix().
5999 *
6000 * This enables us to add/remove arguments and force different levels of
6001 * inlining as we wish.
6002 *
6003 * @return Strict VBox status code.
6004 */
6005#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
6006IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
6007{
6008 NOREF(cbInstr);
6009 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
6010}
6011
6012
6013/**
6014 * Macro for calling iemCImplRaiseInvalidOpcode().
6015 *
6016 * This enables us to add/remove arguments and force different levels of
6017 * inlining as we wish.
6018 *
6019 * @return Strict VBox status code.
6020 */
6021#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
6022IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
6023{
6024 NOREF(cbInstr);
6025 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
6026}
6027
6028
6029/** @} */
6030
6031
6032/*
6033 *
6034 * Helpers routines.
6035 * Helpers routines.
6036 * Helpers routines.
6037 *
6038 */
6039
6040/**
6041 * Recalculates the effective operand size.
6042 *
6043 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6044 */
6045IEM_STATIC void iemRecalEffOpSize(PVMCPUCC pVCpu)
6046{
6047 switch (pVCpu->iem.s.enmCpuMode)
6048 {
6049 case IEMMODE_16BIT:
6050 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
6051 break;
6052 case IEMMODE_32BIT:
6053 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
6054 break;
6055 case IEMMODE_64BIT:
6056 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
6057 {
6058 case 0:
6059 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
6060 break;
6061 case IEM_OP_PRF_SIZE_OP:
6062 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
6063 break;
6064 case IEM_OP_PRF_SIZE_REX_W:
6065 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
6066 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
6067 break;
6068 }
6069 break;
6070 default:
6071 AssertFailed();
6072 }
6073}
6074
6075
6076/**
6077 * Sets the default operand size to 64-bit and recalculates the effective
6078 * operand size.
6079 *
6080 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6081 */
6082IEM_STATIC void iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
6083{
6084 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6085 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
6086 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
6087 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
6088 else
6089 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
6090}
6091
6092
6093/*
6094 *
6095 * Common opcode decoders.
6096 * Common opcode decoders.
6097 * Common opcode decoders.
6098 *
6099 */
6100//#include <iprt/mem.h>
6101
6102/**
6103 * Used to add extra details about a stub case.
6104 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6105 */
6106IEM_STATIC void iemOpStubMsg2(PVMCPUCC pVCpu)
6107{
6108#if defined(LOG_ENABLED) && defined(IN_RING3)
6109 PVM pVM = pVCpu->CTX_SUFF(pVM);
6110 char szRegs[4096];
6111 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
6112 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
6113 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
6114 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
6115 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
6116 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
6117 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
6118 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
6119 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
6120 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
6121 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
6122 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
6123 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
6124 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
6125 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
6126 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
6127 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
6128 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
6129 " efer=%016VR{efer}\n"
6130 " pat=%016VR{pat}\n"
6131 " sf_mask=%016VR{sf_mask}\n"
6132 "krnl_gs_base=%016VR{krnl_gs_base}\n"
6133 " lstar=%016VR{lstar}\n"
6134 " star=%016VR{star} cstar=%016VR{cstar}\n"
6135 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
6136 );
6137
6138 char szInstr[256];
6139 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
6140 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
6141 szInstr, sizeof(szInstr), NULL);
6142
6143 RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
6144#else
6145 RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
6146#endif
6147}
6148
6149/**
6150 * Complains about a stub.
6151 *
6152 * Providing two versions of this macro, one for daily use and one for use when
6153 * working on IEM.
6154 */
6155#if 0
6156# define IEMOP_BITCH_ABOUT_STUB() \
6157 do { \
6158 RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
6159 iemOpStubMsg2(pVCpu); \
6160 RTAssertPanic(); \
6161 } while (0)
6162#else
6163# define IEMOP_BITCH_ABOUT_STUB() Log(("Stub: %s (line %d)\n", __FUNCTION__, __LINE__));
6164#endif
6165
6166/** Stubs an opcode. */
6167#define FNIEMOP_STUB(a_Name) \
6168 FNIEMOP_DEF(a_Name) \
6169 { \
6170 RT_NOREF_PV(pVCpu); \
6171 IEMOP_BITCH_ABOUT_STUB(); \
6172 return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
6173 } \
6174 typedef int ignore_semicolon
6175
6176/** Stubs an opcode. */
6177#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
6178 FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
6179 { \
6180 RT_NOREF_PV(pVCpu); \
6181 RT_NOREF_PV(a_Name0); \
6182 IEMOP_BITCH_ABOUT_STUB(); \
6183 return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
6184 } \
6185 typedef int ignore_semicolon
6186
6187/** Stubs an opcode which currently should raise \#UD. */
6188#define FNIEMOP_UD_STUB(a_Name) \
6189 FNIEMOP_DEF(a_Name) \
6190 { \
6191 Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
6192 return IEMOP_RAISE_INVALID_OPCODE(); \
6193 } \
6194 typedef int ignore_semicolon
6195
6196/** Stubs an opcode which currently should raise \#UD. */
6197#define FNIEMOP_UD_STUB_1(a_Name, a_Type0, a_Name0) \
6198 FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
6199 { \
6200 RT_NOREF_PV(pVCpu); \
6201 RT_NOREF_PV(a_Name0); \
6202 Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
6203 return IEMOP_RAISE_INVALID_OPCODE(); \
6204 } \
6205 typedef int ignore_semicolon
6206
6207
6208
6209/** @name Register Access.
6210 * @{
6211 */
6212
6213/**
6214 * Gets a reference (pointer) to the specified hidden segment register.
6215 *
6216 * @returns Hidden register reference.
6217 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6218 * @param iSegReg The segment register.
6219 */
6220IEM_STATIC PCPUMSELREG iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
6221{
6222 Assert(iSegReg < X86_SREG_COUNT);
6223 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6224 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6225
6226 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
6227 return pSReg;
6228}
6229
6230
6231/**
6232 * Ensures that the given hidden segment register is up to date.
6233 *
6234 * @returns Hidden register reference.
6235 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6236 * @param pSReg The segment register.
6237 */
6238IEM_STATIC PCPUMSELREG iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
6239{
6240 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
6241 NOREF(pVCpu);
6242 return pSReg;
6243}
6244
6245
6246/**
6247 * Gets a reference (pointer) to the specified segment register (the selector
6248 * value).
6249 *
6250 * @returns Pointer to the selector variable.
6251 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6252 * @param iSegReg The segment register.
6253 */
6254DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
6255{
6256 Assert(iSegReg < X86_SREG_COUNT);
6257 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6258 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
6259}
6260
6261
6262/**
6263 * Fetches the selector value of a segment register.
6264 *
6265 * @returns The selector value.
6266 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6267 * @param iSegReg The segment register.
6268 */
6269DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
6270{
6271 Assert(iSegReg < X86_SREG_COUNT);
6272 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6273 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
6274}
6275
6276
6277/**
6278 * Fetches the base address value of a segment register.
6279 *
6280 * @returns The selector value.
6281 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6282 * @param iSegReg The segment register.
6283 */
6284DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
6285{
6286 Assert(iSegReg < X86_SREG_COUNT);
6287 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6288 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6289}
6290
6291
6292/**
6293 * Gets a reference (pointer) to the specified general purpose register.
6294 *
6295 * @returns Register reference.
6296 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6297 * @param iReg The general purpose register.
6298 */
6299DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
6300{
6301 Assert(iReg < 16);
6302 return &pVCpu->cpum.GstCtx.aGRegs[iReg];
6303}
6304
6305
6306/**
6307 * Gets a reference (pointer) to the specified 8-bit general purpose register.
6308 *
6309 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
6310 *
6311 * @returns Register reference.
6312 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6313 * @param iReg The register.
6314 */
6315DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
6316{
6317 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
6318 {
6319 Assert(iReg < 16);
6320 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
6321 }
6322 /* high 8-bit register. */
6323 Assert(iReg < 8);
6324 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
6325}
6326
6327
6328/**
6329 * Gets a reference (pointer) to the specified 16-bit general purpose register.
6330 *
6331 * @returns Register reference.
6332 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6333 * @param iReg The register.
6334 */
6335DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
6336{
6337 Assert(iReg < 16);
6338 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
6339}
6340
6341
6342/**
6343 * Gets a reference (pointer) to the specified 32-bit general purpose register.
6344 *
6345 * @returns Register reference.
6346 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6347 * @param iReg The register.
6348 */
6349DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
6350{
6351 Assert(iReg < 16);
6352 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
6353}
6354
6355
6356/**
6357 * Gets a reference (pointer) to the specified 64-bit general purpose register.
6358 *
6359 * @returns Register reference.
6360 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6361 * @param iReg The register.
6362 */
6363DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
6364{
6365 Assert(iReg < 64);
6366 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
6367}
6368
6369
6370/**
6371 * Gets a reference (pointer) to the specified segment register's base address.
6372 *
6373 * @returns Segment register base address reference.
6374 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6375 * @param iSegReg The segment selector.
6376 */
6377DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
6378{
6379 Assert(iSegReg < X86_SREG_COUNT);
6380 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6381 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6382}
6383
6384
6385/**
6386 * Fetches the value of a 8-bit general purpose register.
6387 *
6388 * @returns The register value.
6389 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6390 * @param iReg The register.
6391 */
6392DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
6393{
6394 return *iemGRegRefU8(pVCpu, iReg);
6395}
6396
6397
6398/**
6399 * Fetches the value of a 16-bit general purpose register.
6400 *
6401 * @returns The register value.
6402 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6403 * @param iReg The register.
6404 */
6405DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
6406{
6407 Assert(iReg < 16);
6408 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
6409}
6410
6411
6412/**
6413 * Fetches the value of a 32-bit general purpose register.
6414 *
6415 * @returns The register value.
6416 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6417 * @param iReg The register.
6418 */
6419DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
6420{
6421 Assert(iReg < 16);
6422 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
6423}
6424
6425
6426/**
6427 * Fetches the value of a 64-bit general purpose register.
6428 *
6429 * @returns The register value.
6430 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6431 * @param iReg The register.
6432 */
6433DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
6434{
6435 Assert(iReg < 16);
6436 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
6437}
6438
6439
6440/**
6441 * Adds a 8-bit signed jump offset to RIP/EIP/IP.
6442 *
6443 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6444 * segment limit.
6445 *
6446 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6447 * @param offNextInstr The offset of the next instruction.
6448 */
6449IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPUCC pVCpu, int8_t offNextInstr)
6450{
6451 switch (pVCpu->iem.s.enmEffOpSize)
6452 {
6453 case IEMMODE_16BIT:
6454 {
6455 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6456 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
6457 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6458 return iemRaiseGeneralProtectionFault0(pVCpu);
6459 pVCpu->cpum.GstCtx.rip = uNewIp;
6460 break;
6461 }
6462
6463 case IEMMODE_32BIT:
6464 {
6465 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6466 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6467
6468 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6469 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
6470 return iemRaiseGeneralProtectionFault0(pVCpu);
6471 pVCpu->cpum.GstCtx.rip = uNewEip;
6472 break;
6473 }
6474
6475 case IEMMODE_64BIT:
6476 {
6477 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6478
6479 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6480 if (!IEM_IS_CANONICAL(uNewRip))
6481 return iemRaiseGeneralProtectionFault0(pVCpu);
6482 pVCpu->cpum.GstCtx.rip = uNewRip;
6483 break;
6484 }
6485
6486 IEM_NOT_REACHED_DEFAULT_CASE_RET();
6487 }
6488
6489 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6490
6491#ifndef IEM_WITH_CODE_TLB
6492 /* Flush the prefetch buffer. */
6493 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6494#endif
6495
6496 return VINF_SUCCESS;
6497}
6498
6499
6500/**
6501 * Adds a 16-bit signed jump offset to RIP/EIP/IP.
6502 *
6503 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6504 * segment limit.
6505 *
6506 * @returns Strict VBox status code.
6507 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6508 * @param offNextInstr The offset of the next instruction.
6509 */
6510IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPUCC pVCpu, int16_t offNextInstr)
6511{
6512 Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
6513
6514 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6515 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
6516 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6517 return iemRaiseGeneralProtectionFault0(pVCpu);
6518 /** @todo Test 16-bit jump in 64-bit mode. possible? */
6519 pVCpu->cpum.GstCtx.rip = uNewIp;
6520 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6521
6522#ifndef IEM_WITH_CODE_TLB
6523 /* Flush the prefetch buffer. */
6524 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6525#endif
6526
6527 return VINF_SUCCESS;
6528}
6529
6530
6531/**
6532 * Adds a 32-bit signed jump offset to RIP/EIP/IP.
6533 *
6534 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6535 * segment limit.
6536 *
6537 * @returns Strict VBox status code.
6538 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6539 * @param offNextInstr The offset of the next instruction.
6540 */
6541IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPUCC pVCpu, int32_t offNextInstr)
6542{
6543 Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
6544
6545 if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
6546 {
6547 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6548
6549 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6550 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
6551 return iemRaiseGeneralProtectionFault0(pVCpu);
6552 pVCpu->cpum.GstCtx.rip = uNewEip;
6553 }
6554 else
6555 {
6556 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6557
6558 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6559 if (!IEM_IS_CANONICAL(uNewRip))
6560 return iemRaiseGeneralProtectionFault0(pVCpu);
6561 pVCpu->cpum.GstCtx.rip = uNewRip;
6562 }
6563 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6564
6565#ifndef IEM_WITH_CODE_TLB
6566 /* Flush the prefetch buffer. */
6567 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6568#endif
6569
6570 return VINF_SUCCESS;
6571}
6572
6573
6574/**
6575 * Performs a near jump to the specified address.
6576 *
6577 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6578 * segment limit.
6579 *
6580 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6581 * @param uNewRip The new RIP value.
6582 */
6583IEM_STATIC VBOXSTRICTRC iemRegRipJump(PVMCPUCC pVCpu, uint64_t uNewRip)
6584{
6585 switch (pVCpu->iem.s.enmEffOpSize)
6586 {
6587 case IEMMODE_16BIT:
6588 {
6589 Assert(uNewRip <= UINT16_MAX);
6590 if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
6591 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6592 return iemRaiseGeneralProtectionFault0(pVCpu);
6593 /** @todo Test 16-bit jump in 64-bit mode. */
6594 pVCpu->cpum.GstCtx.rip = uNewRip;
6595 break;
6596 }
6597
6598 case IEMMODE_32BIT:
6599 {
6600 Assert(uNewRip <= UINT32_MAX);
6601 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6602 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6603
6604 if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
6605 return iemRaiseGeneralProtectionFault0(pVCpu);
6606 pVCpu->cpum.GstCtx.rip = uNewRip;
6607 break;
6608 }
6609
6610 case IEMMODE_64BIT:
6611 {
6612 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6613
6614 if (!IEM_IS_CANONICAL(uNewRip))
6615 return iemRaiseGeneralProtectionFault0(pVCpu);
6616 pVCpu->cpum.GstCtx.rip = uNewRip;
6617 break;
6618 }
6619
6620 IEM_NOT_REACHED_DEFAULT_CASE_RET();
6621 }
6622
6623 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6624
6625#ifndef IEM_WITH_CODE_TLB
6626 /* Flush the prefetch buffer. */
6627 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6628#endif
6629
6630 return VINF_SUCCESS;
6631}
6632
6633
6634/**
6635 * Get the address of the top of the stack.
6636 *
6637 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6638 */
6639DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
6640{
6641 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6642 return pVCpu->cpum.GstCtx.rsp;
6643 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6644 return pVCpu->cpum.GstCtx.esp;
6645 return pVCpu->cpum.GstCtx.sp;
6646}
6647
6648
6649/**
6650 * Updates the RIP/EIP/IP to point to the next instruction.
6651 *
6652 * This function leaves the EFLAGS.RF flag alone.
6653 *
6654 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6655 * @param cbInstr The number of bytes to add.
6656 */
6657IEM_STATIC void iemRegAddToRipKeepRF(PVMCPUCC pVCpu, uint8_t cbInstr)
6658{
6659 switch (pVCpu->iem.s.enmCpuMode)
6660 {
6661 case IEMMODE_16BIT:
6662 Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
6663 pVCpu->cpum.GstCtx.eip += cbInstr;
6664 pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
6665 break;
6666
6667 case IEMMODE_32BIT:
6668 pVCpu->cpum.GstCtx.eip += cbInstr;
6669 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6670 break;
6671
6672 case IEMMODE_64BIT:
6673 pVCpu->cpum.GstCtx.rip += cbInstr;
6674 break;
6675 default: AssertFailed();
6676 }
6677}
6678
6679
6680#if 0
6681/**
6682 * Updates the RIP/EIP/IP to point to the next instruction.
6683 *
6684 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6685 */
6686IEM_STATIC void iemRegUpdateRipKeepRF(PVMCPUCC pVCpu)
6687{
6688 return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
6689}
6690#endif
6691
6692
6693
6694/**
6695 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
6696 *
6697 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6698 * @param cbInstr The number of bytes to add.
6699 */
6700IEM_STATIC void iemRegAddToRipAndClearRF(PVMCPUCC pVCpu, uint8_t cbInstr)
6701{
6702 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6703
6704 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
6705#if ARCH_BITS >= 64
6706 static uint64_t const s_aRipMasks[] = { UINT64_C(0xffffffff), UINT64_C(0xffffffff), UINT64_MAX };
6707 Assert(pVCpu->cpum.GstCtx.rip <= s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode]);
6708 pVCpu->cpum.GstCtx.rip = (pVCpu->cpum.GstCtx.rip + cbInstr) & s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode];
6709#else
6710 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6711 pVCpu->cpum.GstCtx.rip += cbInstr;
6712 else
6713 pVCpu->cpum.GstCtx.eip += cbInstr;
6714#endif
6715}
6716
6717
6718/**
6719 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
6720 *
6721 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6722 */
6723IEM_STATIC void iemRegUpdateRipAndClearRF(PVMCPUCC pVCpu)
6724{
6725 return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
6726}
6727
6728
6729/**
6730 * Adds to the stack pointer.
6731 *
6732 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6733 * @param cbToAdd The number of bytes to add (8-bit!).
6734 */
6735DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
6736{
6737 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6738 pVCpu->cpum.GstCtx.rsp += cbToAdd;
6739 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6740 pVCpu->cpum.GstCtx.esp += cbToAdd;
6741 else
6742 pVCpu->cpum.GstCtx.sp += cbToAdd;
6743}
6744
6745
6746/**
6747 * Subtracts from the stack pointer.
6748 *
6749 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6750 * @param cbToSub The number of bytes to subtract (8-bit!).
6751 */
6752DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
6753{
6754 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6755 pVCpu->cpum.GstCtx.rsp -= cbToSub;
6756 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6757 pVCpu->cpum.GstCtx.esp -= cbToSub;
6758 else
6759 pVCpu->cpum.GstCtx.sp -= cbToSub;
6760}
6761
6762
6763/**
6764 * Adds to the temporary stack pointer.
6765 *
6766 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6767 * @param pTmpRsp The temporary SP/ESP/RSP to update.
6768 * @param cbToAdd The number of bytes to add (16-bit).
6769 */
6770DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
6771{
6772 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6773 pTmpRsp->u += cbToAdd;
6774 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6775 pTmpRsp->DWords.dw0 += cbToAdd;
6776 else
6777 pTmpRsp->Words.w0 += cbToAdd;
6778}
6779
6780
6781/**
6782 * Subtracts from the temporary stack pointer.
6783 *
6784 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6785 * @param pTmpRsp The temporary SP/ESP/RSP to update.
6786 * @param cbToSub The number of bytes to subtract.
6787 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
6788 * expecting that.
6789 */
6790DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
6791{
6792 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6793 pTmpRsp->u -= cbToSub;
6794 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6795 pTmpRsp->DWords.dw0 -= cbToSub;
6796 else
6797 pTmpRsp->Words.w0 -= cbToSub;
6798}
6799
6800
6801/**
6802 * Calculates the effective stack address for a push of the specified size as
6803 * well as the new RSP value (upper bits may be masked).
6804 *
6805 * @returns Effective stack addressf for the push.
6806 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6807 * @param cbItem The size of the stack item to pop.
6808 * @param puNewRsp Where to return the new RSP value.
6809 */
6810DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
6811{
6812 RTUINT64U uTmpRsp;
6813 RTGCPTR GCPtrTop;
6814 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
6815
6816 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6817 GCPtrTop = uTmpRsp.u -= cbItem;
6818 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6819 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
6820 else
6821 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
6822 *puNewRsp = uTmpRsp.u;
6823 return GCPtrTop;
6824}
6825
6826
6827/**
6828 * Gets the current stack pointer and calculates the value after a pop of the
6829 * specified size.
6830 *
6831 * @returns Current stack pointer.
6832 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6833 * @param cbItem The size of the stack item to pop.
6834 * @param puNewRsp Where to return the new RSP value.
6835 */
6836DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
6837{
6838 RTUINT64U uTmpRsp;
6839 RTGCPTR GCPtrTop;
6840 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
6841
6842 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6843 {
6844 GCPtrTop = uTmpRsp.u;
6845 uTmpRsp.u += cbItem;
6846 }
6847 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6848 {
6849 GCPtrTop = uTmpRsp.DWords.dw0;
6850 uTmpRsp.DWords.dw0 += cbItem;
6851 }
6852 else
6853 {
6854 GCPtrTop = uTmpRsp.Words.w0;
6855 uTmpRsp.Words.w0 += cbItem;
6856 }
6857 *puNewRsp = uTmpRsp.u;
6858 return GCPtrTop;
6859}
6860
6861
6862/**
6863 * Calculates the effective stack address for a push of the specified size as
6864 * well as the new temporary RSP value (upper bits may be masked).
6865 *
6866 * @returns Effective stack addressf for the push.
6867 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6868 * @param pTmpRsp The temporary stack pointer. This is updated.
6869 * @param cbItem The size of the stack item to pop.
6870 */
6871DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
6872{
6873 RTGCPTR GCPtrTop;
6874
6875 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6876 GCPtrTop = pTmpRsp->u -= cbItem;
6877 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6878 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
6879 else
6880 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
6881 return GCPtrTop;
6882}
6883
6884
6885/**
6886 * Gets the effective stack address for a pop of the specified size and
6887 * calculates and updates the temporary RSP.
6888 *
6889 * @returns Current stack pointer.
6890 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6891 * @param pTmpRsp The temporary stack pointer. This is updated.
6892 * @param cbItem The size of the stack item to pop.
6893 */
6894DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
6895{
6896 RTGCPTR GCPtrTop;
6897 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6898 {
6899 GCPtrTop = pTmpRsp->u;
6900 pTmpRsp->u += cbItem;
6901 }
6902 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6903 {
6904 GCPtrTop = pTmpRsp->DWords.dw0;
6905 pTmpRsp->DWords.dw0 += cbItem;
6906 }
6907 else
6908 {
6909 GCPtrTop = pTmpRsp->Words.w0;
6910 pTmpRsp->Words.w0 += cbItem;
6911 }
6912 return GCPtrTop;
6913}
6914
6915/** @} */
6916
6917
6918/** @name FPU access and helpers.
6919 *
6920 * @{
6921 */
6922
6923
6924/**
6925 * Hook for preparing to use the host FPU.
6926 *
6927 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6928 *
6929 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6930 */
6931DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
6932{
6933#ifdef IN_RING3
6934 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6935#else
6936 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
6937#endif
6938 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6939}
6940
6941
6942/**
6943 * Hook for preparing to use the host FPU for SSE.
6944 *
6945 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6946 *
6947 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6948 */
6949DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
6950{
6951 iemFpuPrepareUsage(pVCpu);
6952}
6953
6954
6955/**
6956 * Hook for preparing to use the host FPU for AVX.
6957 *
6958 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6959 *
6960 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6961 */
6962DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
6963{
6964 iemFpuPrepareUsage(pVCpu);
6965}
6966
6967
6968/**
6969 * Hook for actualizing the guest FPU state before the interpreter reads it.
6970 *
6971 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6972 *
6973 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6974 */
6975DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
6976{
6977#ifdef IN_RING3
6978 NOREF(pVCpu);
6979#else
6980 CPUMRZFpuStateActualizeForRead(pVCpu);
6981#endif
6982 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6983}
6984
6985
6986/**
6987 * Hook for actualizing the guest FPU state before the interpreter changes it.
6988 *
6989 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6990 *
6991 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6992 */
6993DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
6994{
6995#ifdef IN_RING3
6996 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6997#else
6998 CPUMRZFpuStateActualizeForChange(pVCpu);
6999#endif
7000 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7001}
7002
7003
7004/**
7005 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
7006 * only.
7007 *
7008 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7009 *
7010 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7011 */
7012DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
7013{
7014#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
7015 NOREF(pVCpu);
7016#else
7017 CPUMRZFpuStateActualizeSseForRead(pVCpu);
7018#endif
7019 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7020}
7021
7022
7023/**
7024 * Hook for actualizing the guest XMM0..15 and MXCSR register state for
7025 * read+write.
7026 *
7027 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7028 *
7029 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7030 */
7031DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
7032{
7033#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
7034 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
7035#else
7036 CPUMRZFpuStateActualizeForChange(pVCpu);
7037#endif
7038 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7039
7040 /* Make sure any changes are loaded the next time around. */
7041 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE;
7042}
7043
7044
7045/**
7046 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
7047 * only.
7048 *
7049 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7050 *
7051 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7052 */
7053DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
7054{
7055#ifdef IN_RING3
7056 NOREF(pVCpu);
7057#else
7058 CPUMRZFpuStateActualizeAvxForRead(pVCpu);
7059#endif
7060 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7061}
7062
7063
7064/**
7065 * Hook for actualizing the guest YMM0..15 and MXCSR register state for
7066 * read+write.
7067 *
7068 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7069 *
7070 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7071 */
7072DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
7073{
7074#ifdef IN_RING3
7075 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
7076#else
7077 CPUMRZFpuStateActualizeForChange(pVCpu);
7078#endif
7079 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7080
7081 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */
7082 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;
7083}
7084
7085
7086/**
7087 * Stores a QNaN value into a FPU register.
7088 *
7089 * @param pReg Pointer to the register.
7090 */
7091DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
7092{
7093 pReg->au32[0] = UINT32_C(0x00000000);
7094 pReg->au32[1] = UINT32_C(0xc0000000);
7095 pReg->au16[4] = UINT16_C(0xffff);
7096}
7097
7098
7099/**
7100 * Updates the FOP, FPU.CS and FPUIP registers.
7101 *
7102 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7103 * @param pFpuCtx The FPU context.
7104 */
7105DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
7106{
7107 Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
7108 pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
7109 /** @todo x87.CS and FPUIP needs to be kept seperately. */
7110 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
7111 {
7112 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
7113 * happens in real mode here based on the fnsave and fnstenv images. */
7114 pFpuCtx->CS = 0;
7115 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
7116 }
7117 else if (!IEM_IS_LONG_MODE(pVCpu))
7118 {
7119 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
7120 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
7121 }
7122 else
7123 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
7124}
7125
7126
7127/**
7128 * Updates the x87.DS and FPUDP registers.
7129 *
7130 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7131 * @param pFpuCtx The FPU context.
7132 * @param iEffSeg The effective segment register.
7133 * @param GCPtrEff The effective address relative to @a iEffSeg.
7134 */
7135DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7136{
7137 RTSEL sel;
7138 switch (iEffSeg)
7139 {
7140 case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
7141 case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
7142 case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
7143 case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
7144 case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
7145 case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
7146 default:
7147 AssertMsgFailed(("%d\n", iEffSeg));
7148 sel = pVCpu->cpum.GstCtx.ds.Sel;
7149 }
7150 /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
7151 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
7152 {
7153 pFpuCtx->DS = 0;
7154 pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
7155 }
7156 else if (!IEM_IS_LONG_MODE(pVCpu))
7157 {
7158 pFpuCtx->DS = sel;
7159 pFpuCtx->FPUDP = GCPtrEff;
7160 }
7161 else
7162 *(uint64_t *)&pFpuCtx->FPUDP = GCPtrEff;
7163}
7164
7165
7166/**
7167 * Rotates the stack registers in the push direction.
7168 *
7169 * @param pFpuCtx The FPU context.
7170 * @remarks This is a complete waste of time, but fxsave stores the registers in
7171 * stack order.
7172 */
7173DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
7174{
7175 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
7176 pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
7177 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
7178 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
7179 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
7180 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
7181 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
7182 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
7183 pFpuCtx->aRegs[0].r80 = r80Tmp;
7184}
7185
7186
7187/**
7188 * Rotates the stack registers in the pop direction.
7189 *
7190 * @param pFpuCtx The FPU context.
7191 * @remarks This is a complete waste of time, but fxsave stores the registers in
7192 * stack order.
7193 */
7194DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
7195{
7196 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
7197 pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
7198 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
7199 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
7200 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
7201 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
7202 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
7203 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
7204 pFpuCtx->aRegs[7].r80 = r80Tmp;
7205}
7206
7207
7208/**
7209 * Updates FSW and pushes a FPU result onto the FPU stack if no pending
7210 * exception prevents it.
7211 *
7212 * @param pResult The FPU operation result to push.
7213 * @param pFpuCtx The FPU context.
7214 */
7215IEM_STATIC void iemFpuMaybePushResult(PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx)
7216{
7217 /* Update FSW and bail if there are pending exceptions afterwards. */
7218 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
7219 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
7220 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7221 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7222 {
7223 pFpuCtx->FSW = fFsw;
7224 return;
7225 }
7226
7227 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
7228 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
7229 {
7230 /* All is fine, push the actual value. */
7231 pFpuCtx->FTW |= RT_BIT(iNewTop);
7232 pFpuCtx->aRegs[7].r80 = pResult->r80Result;
7233 }
7234 else if (pFpuCtx->FCW & X86_FCW_IM)
7235 {
7236 /* Masked stack overflow, push QNaN. */
7237 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
7238 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7239 }
7240 else
7241 {
7242 /* Raise stack overflow, don't push anything. */
7243 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
7244 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
7245 return;
7246 }
7247
7248 fFsw &= ~X86_FSW_TOP_MASK;
7249 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
7250 pFpuCtx->FSW = fFsw;
7251
7252 iemFpuRotateStackPush(pFpuCtx);
7253}
7254
7255
7256/**
7257 * Stores a result in a FPU register and updates the FSW and FTW.
7258 *
7259 * @param pFpuCtx The FPU context.
7260 * @param pResult The result to store.
7261 * @param iStReg Which FPU register to store it in.
7262 */
7263IEM_STATIC void iemFpuStoreResultOnly(PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg)
7264{
7265 Assert(iStReg < 8);
7266 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7267 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7268 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_TOP_MASK;
7269 pFpuCtx->FTW |= RT_BIT(iReg);
7270 pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
7271}
7272
7273
7274/**
7275 * Only updates the FPU status word (FSW) with the result of the current
7276 * instruction.
7277 *
7278 * @param pFpuCtx The FPU context.
7279 * @param u16FSW The FSW output of the current instruction.
7280 */
7281IEM_STATIC void iemFpuUpdateFSWOnly(PX86FXSTATE pFpuCtx, uint16_t u16FSW)
7282{
7283 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7284 pFpuCtx->FSW |= u16FSW & ~X86_FSW_TOP_MASK;
7285}
7286
7287
7288/**
7289 * Pops one item off the FPU stack if no pending exception prevents it.
7290 *
7291 * @param pFpuCtx The FPU context.
7292 */
7293IEM_STATIC void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx)
7294{
7295 /* Check pending exceptions. */
7296 uint16_t uFSW = pFpuCtx->FSW;
7297 if ( (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7298 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7299 return;
7300
7301 /* TOP--. */
7302 uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
7303 uFSW &= ~X86_FSW_TOP_MASK;
7304 uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7305 pFpuCtx->FSW = uFSW;
7306
7307 /* Mark the previous ST0 as empty. */
7308 iOldTop >>= X86_FSW_TOP_SHIFT;
7309 pFpuCtx->FTW &= ~RT_BIT(iOldTop);
7310
7311 /* Rotate the registers. */
7312 iemFpuRotateStackPop(pFpuCtx);
7313}
7314
7315
7316/**
7317 * Pushes a FPU result onto the FPU stack if no pending exception prevents it.
7318 *
7319 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7320 * @param pResult The FPU operation result to push.
7321 */
7322IEM_STATIC void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult)
7323{
7324 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7325 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7326 iemFpuMaybePushResult(pResult, pFpuCtx);
7327}
7328
7329
7330/**
7331 * Pushes a FPU result onto the FPU stack if no pending exception prevents it,
7332 * and sets FPUDP and FPUDS.
7333 *
7334 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7335 * @param pResult The FPU operation result to push.
7336 * @param iEffSeg The effective segment register.
7337 * @param GCPtrEff The effective address relative to @a iEffSeg.
7338 */
7339IEM_STATIC void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7340{
7341 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7342 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7343 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7344 iemFpuMaybePushResult(pResult, pFpuCtx);
7345}
7346
7347
7348/**
7349 * Replace ST0 with the first value and push the second onto the FPU stack,
7350 * unless a pending exception prevents it.
7351 *
7352 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7353 * @param pResult The FPU operation result to store and push.
7354 */
7355IEM_STATIC void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult)
7356{
7357 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7358 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7359
7360 /* Update FSW and bail if there are pending exceptions afterwards. */
7361 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
7362 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
7363 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7364 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7365 {
7366 pFpuCtx->FSW = fFsw;
7367 return;
7368 }
7369
7370 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
7371 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
7372 {
7373 /* All is fine, push the actual value. */
7374 pFpuCtx->FTW |= RT_BIT(iNewTop);
7375 pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
7376 pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
7377 }
7378 else if (pFpuCtx->FCW & X86_FCW_IM)
7379 {
7380 /* Masked stack overflow, push QNaN. */
7381 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
7382 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
7383 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7384 }
7385 else
7386 {
7387 /* Raise stack overflow, don't push anything. */
7388 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
7389 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
7390 return;
7391 }
7392
7393 fFsw &= ~X86_FSW_TOP_MASK;
7394 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
7395 pFpuCtx->FSW = fFsw;
7396
7397 iemFpuRotateStackPush(pFpuCtx);
7398}
7399
7400
7401/**
7402 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
7403 * FOP.
7404 *
7405 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7406 * @param pResult The result to store.
7407 * @param iStReg Which FPU register to store it in.
7408 */
7409IEM_STATIC void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
7410{
7411 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7412 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7413 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7414}
7415
7416
7417/**
7418 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
7419 * FOP, and then pops the stack.
7420 *
7421 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7422 * @param pResult The result to store.
7423 * @param iStReg Which FPU register to store it in.
7424 */
7425IEM_STATIC void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
7426{
7427 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7428 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7429 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7430 iemFpuMaybePopOne(pFpuCtx);
7431}
7432
7433
7434/**
7435 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
7436 * FPUDP, and FPUDS.
7437 *
7438 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7439 * @param pResult The result to store.
7440 * @param iStReg Which FPU register to store it in.
7441 * @param iEffSeg The effective memory operand selector register.
7442 * @param GCPtrEff The effective memory operand offset.
7443 */
7444IEM_STATIC void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
7445 uint8_t iEffSeg, RTGCPTR GCPtrEff)
7446{
7447 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7448 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7449 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7450 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7451}
7452
7453
7454/**
7455 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
7456 * FPUDP, and FPUDS, and then pops the stack.
7457 *
7458 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7459 * @param pResult The result to store.
7460 * @param iStReg Which FPU register to store it in.
7461 * @param iEffSeg The effective memory operand selector register.
7462 * @param GCPtrEff The effective memory operand offset.
7463 */
7464IEM_STATIC void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
7465 uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7466{
7467 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7468 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7469 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7470 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7471 iemFpuMaybePopOne(pFpuCtx);
7472}
7473
7474
7475/**
7476 * Updates the FOP, FPUIP, and FPUCS. For FNOP.
7477 *
7478 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7479 */
7480IEM_STATIC void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu)
7481{
7482 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7483 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7484}
7485
7486
7487/**
7488 * Marks the specified stack register as free (for FFREE).
7489 *
7490 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7491 * @param iStReg The register to free.
7492 */
7493IEM_STATIC void iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
7494{
7495 Assert(iStReg < 8);
7496 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7497 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7498 pFpuCtx->FTW &= ~RT_BIT(iReg);
7499}
7500
7501
7502/**
7503 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
7504 *
7505 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7506 */
7507IEM_STATIC void iemFpuStackIncTop(PVMCPUCC pVCpu)
7508{
7509 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7510 uint16_t uFsw = pFpuCtx->FSW;
7511 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
7512 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7513 uFsw &= ~X86_FSW_TOP_MASK;
7514 uFsw |= uTop;
7515 pFpuCtx->FSW = uFsw;
7516}
7517
7518
7519/**
7520 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
7521 *
7522 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7523 */
7524IEM_STATIC void iemFpuStackDecTop(PVMCPUCC pVCpu)
7525{
7526 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7527 uint16_t uFsw = pFpuCtx->FSW;
7528 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
7529 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7530 uFsw &= ~X86_FSW_TOP_MASK;
7531 uFsw |= uTop;
7532 pFpuCtx->FSW = uFsw;
7533}
7534
7535
7536/**
7537 * Updates the FSW, FOP, FPUIP, and FPUCS.
7538 *
7539 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7540 * @param u16FSW The FSW from the current instruction.
7541 */
7542IEM_STATIC void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW)
7543{
7544 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7545 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7546 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7547}
7548
7549
7550/**
7551 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
7552 *
7553 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7554 * @param u16FSW The FSW from the current instruction.
7555 */
7556IEM_STATIC void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW)
7557{
7558 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7559 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7560 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7561 iemFpuMaybePopOne(pFpuCtx);
7562}
7563
7564
7565/**
7566 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
7567 *
7568 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7569 * @param u16FSW The FSW from the current instruction.
7570 * @param iEffSeg The effective memory operand selector register.
7571 * @param GCPtrEff The effective memory operand offset.
7572 */
7573IEM_STATIC void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7574{
7575 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7576 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7577 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7578 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7579}
7580
7581
7582/**
7583 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
7584 *
7585 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7586 * @param u16FSW The FSW from the current instruction.
7587 */
7588IEM_STATIC void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW)
7589{
7590 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7591 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7592 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7593 iemFpuMaybePopOne(pFpuCtx);
7594 iemFpuMaybePopOne(pFpuCtx);
7595}
7596
7597
7598/**
7599 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
7600 *
7601 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7602 * @param u16FSW The FSW from the current instruction.
7603 * @param iEffSeg The effective memory operand selector register.
7604 * @param GCPtrEff The effective memory operand offset.
7605 */
7606IEM_STATIC void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7607{
7608 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7609 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7610 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7611 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7612 iemFpuMaybePopOne(pFpuCtx);
7613}
7614
7615
7616/**
7617 * Worker routine for raising an FPU stack underflow exception.
7618 *
7619 * @param pFpuCtx The FPU context.
7620 * @param iStReg The stack register being accessed.
7621 */
7622IEM_STATIC void iemFpuStackUnderflowOnly(PX86FXSTATE pFpuCtx, uint8_t iStReg)
7623{
7624 Assert(iStReg < 8 || iStReg == UINT8_MAX);
7625 if (pFpuCtx->FCW & X86_FCW_IM)
7626 {
7627 /* Masked underflow. */
7628 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7629 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7630 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7631 if (iStReg != UINT8_MAX)
7632 {
7633 pFpuCtx->FTW |= RT_BIT(iReg);
7634 iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
7635 }
7636 }
7637 else
7638 {
7639 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7640 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7641 }
7642}
7643
7644
7645/**
7646 * Raises a FPU stack underflow exception.
7647 *
7648 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7649 * @param iStReg The destination register that should be loaded
7650 * with QNaN if \#IS is not masked. Specify
7651 * UINT8_MAX if none (like for fcom).
7652 */
7653DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg)
7654{
7655 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7656 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7657 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7658}
7659
7660
7661DECL_NO_INLINE(IEM_STATIC, void)
7662iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7663{
7664 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7665 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7666 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7667 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7668}
7669
7670
7671DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg)
7672{
7673 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7674 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7675 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7676 iemFpuMaybePopOne(pFpuCtx);
7677}
7678
7679
7680DECL_NO_INLINE(IEM_STATIC, void)
7681iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7682{
7683 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7684 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7685 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7686 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7687 iemFpuMaybePopOne(pFpuCtx);
7688}
7689
7690
7691DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu)
7692{
7693 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7694 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7695 iemFpuStackUnderflowOnly(pFpuCtx, UINT8_MAX);
7696 iemFpuMaybePopOne(pFpuCtx);
7697 iemFpuMaybePopOne(pFpuCtx);
7698}
7699
7700
7701DECL_NO_INLINE(IEM_STATIC, void)
7702iemFpuStackPushUnderflow(PVMCPUCC pVCpu)
7703{
7704 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7705 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7706
7707 if (pFpuCtx->FCW & X86_FCW_IM)
7708 {
7709 /* Masked overflow - Push QNaN. */
7710 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7711 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7712 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7713 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7714 pFpuCtx->FTW |= RT_BIT(iNewTop);
7715 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7716 iemFpuRotateStackPush(pFpuCtx);
7717 }
7718 else
7719 {
7720 /* Exception pending - don't change TOP or the register stack. */
7721 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7722 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7723 }
7724}
7725
7726
7727DECL_NO_INLINE(IEM_STATIC, void)
7728iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu)
7729{
7730 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7731 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7732
7733 if (pFpuCtx->FCW & X86_FCW_IM)
7734 {
7735 /* Masked overflow - Push QNaN. */
7736 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7737 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7738 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7739 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7740 pFpuCtx->FTW |= RT_BIT(iNewTop);
7741 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
7742 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7743 iemFpuRotateStackPush(pFpuCtx);
7744 }
7745 else
7746 {
7747 /* Exception pending - don't change TOP or the register stack. */
7748 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7749 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7750 }
7751}
7752
7753
7754/**
7755 * Worker routine for raising an FPU stack overflow exception on a push.
7756 *
7757 * @param pFpuCtx The FPU context.
7758 */
7759IEM_STATIC void iemFpuStackPushOverflowOnly(PX86FXSTATE pFpuCtx)
7760{
7761 if (pFpuCtx->FCW & X86_FCW_IM)
7762 {
7763 /* Masked overflow. */
7764 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7765 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7766 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
7767 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7768 pFpuCtx->FTW |= RT_BIT(iNewTop);
7769 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7770 iemFpuRotateStackPush(pFpuCtx);
7771 }
7772 else
7773 {
7774 /* Exception pending - don't change TOP or the register stack. */
7775 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7776 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7777 }
7778}
7779
7780
7781/**
7782 * Raises a FPU stack overflow exception on a push.
7783 *
7784 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7785 */
7786DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackPushOverflow(PVMCPUCC pVCpu)
7787{
7788 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7789 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7790 iemFpuStackPushOverflowOnly(pFpuCtx);
7791}
7792
7793
7794/**
7795 * Raises a FPU stack overflow exception on a push with a memory operand.
7796 *
7797 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7798 * @param iEffSeg The effective memory operand selector register.
7799 * @param GCPtrEff The effective memory operand offset.
7800 */
7801DECL_NO_INLINE(IEM_STATIC, void)
7802iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7803{
7804 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7805 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7806 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7807 iemFpuStackPushOverflowOnly(pFpuCtx);
7808}
7809
7810
7811IEM_STATIC int iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
7812{
7813 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7814 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7815 if (pFpuCtx->FTW & RT_BIT(iReg))
7816 return VINF_SUCCESS;
7817 return VERR_NOT_FOUND;
7818}
7819
7820
7821IEM_STATIC int iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
7822{
7823 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7824 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7825 if (pFpuCtx->FTW & RT_BIT(iReg))
7826 {
7827 *ppRef = &pFpuCtx->aRegs[iStReg].r80;
7828 return VINF_SUCCESS;
7829 }
7830 return VERR_NOT_FOUND;
7831}
7832
7833
7834IEM_STATIC int iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
7835 uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
7836{
7837 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7838 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7839 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
7840 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
7841 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
7842 {
7843 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
7844 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
7845 return VINF_SUCCESS;
7846 }
7847 return VERR_NOT_FOUND;
7848}
7849
7850
7851IEM_STATIC int iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
7852{
7853 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
7854 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7855 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
7856 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
7857 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
7858 {
7859 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
7860 return VINF_SUCCESS;
7861 }
7862 return VERR_NOT_FOUND;
7863}
7864
7865
7866/**
7867 * Updates the FPU exception status after FCW is changed.
7868 *
7869 * @param pFpuCtx The FPU context.
7870 */
7871IEM_STATIC void iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
7872{
7873 uint16_t u16Fsw = pFpuCtx->FSW;
7874 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
7875 u16Fsw |= X86_FSW_ES | X86_FSW_B;
7876 else
7877 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
7878 pFpuCtx->FSW = u16Fsw;
7879}
7880
7881
7882/**
7883 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
7884 *
7885 * @returns The full FTW.
7886 * @param pFpuCtx The FPU context.
7887 */
7888IEM_STATIC uint16_t iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
7889{
7890 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
7891 uint16_t u16Ftw = 0;
7892 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7893 for (unsigned iSt = 0; iSt < 8; iSt++)
7894 {
7895 unsigned const iReg = (iSt + iTop) & 7;
7896 if (!(u8Ftw & RT_BIT(iReg)))
7897 u16Ftw |= 3 << (iReg * 2); /* empty */
7898 else
7899 {
7900 uint16_t uTag;
7901 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
7902 if (pr80Reg->s.uExponent == 0x7fff)
7903 uTag = 2; /* Exponent is all 1's => Special. */
7904 else if (pr80Reg->s.uExponent == 0x0000)
7905 {
7906 if (pr80Reg->s.u64Mantissa == 0x0000)
7907 uTag = 1; /* All bits are zero => Zero. */
7908 else
7909 uTag = 2; /* Must be special. */
7910 }
7911 else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
7912 uTag = 0; /* Valid. */
7913 else
7914 uTag = 2; /* Must be special. */
7915
7916 u16Ftw |= uTag << (iReg * 2); /* empty */
7917 }
7918 }
7919
7920 return u16Ftw;
7921}
7922
7923
7924/**
7925 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
7926 *
7927 * @returns The compressed FTW.
7928 * @param u16FullFtw The full FTW to convert.
7929 */
7930IEM_STATIC uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
7931{
7932 uint8_t u8Ftw = 0;
7933 for (unsigned i = 0; i < 8; i++)
7934 {
7935 if ((u16FullFtw & 3) != 3 /*empty*/)
7936 u8Ftw |= RT_BIT(i);
7937 u16FullFtw >>= 2;
7938 }
7939
7940 return u8Ftw;
7941}
7942
7943/** @} */
7944
7945
7946/** @name Memory access.
7947 *
7948 * @{
7949 */
7950
7951
7952/**
7953 * Updates the IEMCPU::cbWritten counter if applicable.
7954 *
7955 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7956 * @param fAccess The access being accounted for.
7957 * @param cbMem The access size.
7958 */
7959DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
7960{
7961 if ( (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE)
7962 || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA | IEM_ACCESS_TYPE_WRITE) )
7963 pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
7964}
7965
7966
7967/**
7968 * Checks if the given segment can be written to, raise the appropriate
7969 * exception if not.
7970 *
7971 * @returns VBox strict status code.
7972 *
7973 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7974 * @param pHid Pointer to the hidden register.
7975 * @param iSegReg The register number.
7976 * @param pu64BaseAddr Where to return the base address to use for the
7977 * segment. (In 64-bit code it may differ from the
7978 * base in the hidden segment.)
7979 */
7980IEM_STATIC VBOXSTRICTRC
7981iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
7982{
7983 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
7984
7985 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7986 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
7987 else
7988 {
7989 if (!pHid->Attr.n.u1Present)
7990 {
7991 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
7992 AssertRelease(uSel == 0);
7993 Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
7994 return iemRaiseGeneralProtectionFault0(pVCpu);
7995 }
7996
7997 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
7998 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
7999 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
8000 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
8001 *pu64BaseAddr = pHid->u64Base;
8002 }
8003 return VINF_SUCCESS;
8004}
8005
8006
8007/**
8008 * Checks if the given segment can be read from, raise the appropriate
8009 * exception if not.
8010 *
8011 * @returns VBox strict status code.
8012 *
8013 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8014 * @param pHid Pointer to the hidden register.
8015 * @param iSegReg The register number.
8016 * @param pu64BaseAddr Where to return the base address to use for the
8017 * segment. (In 64-bit code it may differ from the
8018 * base in the hidden segment.)
8019 */
8020IEM_STATIC VBOXSTRICTRC
8021iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
8022{
8023 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
8024
8025 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
8026 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
8027 else
8028 {
8029 if (!pHid->Attr.n.u1Present)
8030 {
8031 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
8032 AssertRelease(uSel == 0);
8033 Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
8034 return iemRaiseGeneralProtectionFault0(pVCpu);
8035 }
8036
8037 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
8038 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
8039 *pu64BaseAddr = pHid->u64Base;
8040 }
8041 return VINF_SUCCESS;
8042}
8043
8044
8045/**
8046 * Applies the segment limit, base and attributes.
8047 *
8048 * This may raise a \#GP or \#SS.
8049 *
8050 * @returns VBox strict status code.
8051 *
8052 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8053 * @param fAccess The kind of access which is being performed.
8054 * @param iSegReg The index of the segment register to apply.
8055 * This is UINT8_MAX if none (for IDT, GDT, LDT,
8056 * TSS, ++).
8057 * @param cbMem The access size.
8058 * @param pGCPtrMem Pointer to the guest memory address to apply
8059 * segmentation to. Input and output parameter.
8060 */
8061IEM_STATIC VBOXSTRICTRC
8062iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem)
8063{
8064 if (iSegReg == UINT8_MAX)
8065 return VINF_SUCCESS;
8066
8067 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
8068 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
8069 switch (pVCpu->iem.s.enmCpuMode)
8070 {
8071 case IEMMODE_16BIT:
8072 case IEMMODE_32BIT:
8073 {
8074 RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
8075 RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
8076
8077 if ( pSel->Attr.n.u1Present
8078 && !pSel->Attr.n.u1Unusable)
8079 {
8080 Assert(pSel->Attr.n.u1DescType);
8081 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
8082 {
8083 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8084 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
8085 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
8086
8087 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
8088 {
8089 /** @todo CPL check. */
8090 }
8091
8092 /*
8093 * There are two kinds of data selectors, normal and expand down.
8094 */
8095 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
8096 {
8097 if ( GCPtrFirst32 > pSel->u32Limit
8098 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
8099 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8100 }
8101 else
8102 {
8103 /*
8104 * The upper boundary is defined by the B bit, not the G bit!
8105 */
8106 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
8107 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
8108 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8109 }
8110 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
8111 }
8112 else
8113 {
8114
8115 /*
8116 * Code selector and usually be used to read thru, writing is
8117 * only permitted in real and V8086 mode.
8118 */
8119 if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8120 || ( (fAccess & IEM_ACCESS_TYPE_READ)
8121 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
8122 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
8123 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
8124
8125 if ( GCPtrFirst32 > pSel->u32Limit
8126 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
8127 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8128
8129 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
8130 {
8131 /** @todo CPL check. */
8132 }
8133
8134 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
8135 }
8136 }
8137 else
8138 return iemRaiseGeneralProtectionFault0(pVCpu);
8139 return VINF_SUCCESS;
8140 }
8141
8142 case IEMMODE_64BIT:
8143 {
8144 RTGCPTR GCPtrMem = *pGCPtrMem;
8145 if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS)
8146 *pGCPtrMem = GCPtrMem + pSel->u64Base;
8147
8148 Assert(cbMem >= 1);
8149 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
8150 return VINF_SUCCESS;
8151 /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
8152 * 4.12.2 "Data Limit Checks in 64-bit Mode". */
8153 return iemRaiseGeneralProtectionFault0(pVCpu);
8154 }
8155
8156 default:
8157 AssertFailedReturn(VERR_IEM_IPE_7);
8158 }
8159}
8160
8161
8162/**
8163 * Translates a virtual address to a physical physical address and checks if we
8164 * can access the page as specified.
8165 *
8166 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8167 * @param GCPtrMem The virtual address.
8168 * @param fAccess The intended access.
8169 * @param pGCPhysMem Where to return the physical address.
8170 */
8171IEM_STATIC VBOXSTRICTRC
8172iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem)
8173{
8174 /** @todo Need a different PGM interface here. We're currently using
8175 * generic / REM interfaces. this won't cut it for R0. */
8176 /** @todo If/when PGM handles paged real-mode, we can remove the hack in
8177 * iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
8178 * here. */
8179 PGMPTWALK Walk;
8180 int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
8181 if (RT_FAILURE(rc))
8182 {
8183 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
8184 /** @todo Check unassigned memory in unpaged mode. */
8185 /** @todo Reserved bits in page tables. Requires new PGM interface. */
8186#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
8187 if (Walk.fFailed & PGM_WALKFAIL_EPT)
8188 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
8189#endif
8190 *pGCPhysMem = NIL_RTGCPHYS;
8191 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
8192 }
8193
8194 /* If the page is writable and does not have the no-exec bit set, all
8195 access is allowed. Otherwise we'll have to check more carefully... */
8196 if ((Walk.fEffective & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US))
8197 {
8198 /* Write to read only memory? */
8199 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8200 && !(Walk.fEffective & X86_PTE_RW)
8201 && ( ( pVCpu->iem.s.uCpl == 3
8202 && !(fAccess & IEM_ACCESS_WHAT_SYS))
8203 || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
8204 {
8205 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
8206 *pGCPhysMem = NIL_RTGCPHYS;
8207#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
8208 if (Walk.fFailed & PGM_WALKFAIL_EPT)
8209 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
8210#endif
8211 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
8212 }
8213
8214 /* Kernel memory accessed by userland? */
8215 if ( !(Walk.fEffective & X86_PTE_US)
8216 && pVCpu->iem.s.uCpl == 3
8217 && !(fAccess & IEM_ACCESS_WHAT_SYS))
8218 {
8219 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
8220 *pGCPhysMem = NIL_RTGCPHYS;
8221#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
8222 if (Walk.fFailed & PGM_WALKFAIL_EPT)
8223 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
8224#endif
8225 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
8226 }
8227
8228 /* Executing non-executable memory? */
8229 if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
8230 && (Walk.fEffective & X86_PTE_PAE_NX)
8231 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
8232 {
8233 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
8234 *pGCPhysMem = NIL_RTGCPHYS;
8235#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
8236 if (Walk.fFailed & PGM_WALKFAIL_EPT)
8237 IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
8238#endif
8239 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE),
8240 VERR_ACCESS_DENIED);
8241 }
8242 }
8243
8244 /*
8245 * Set the dirty / access flags.
8246 * ASSUMES this is set when the address is translated rather than on committ...
8247 */
8248 /** @todo testcase: check when A and D bits are actually set by the CPU. */
8249 uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
8250 if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
8251 {
8252 int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
8253 AssertRC(rc2);
8254 /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
8255 Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
8256 }
8257
8258 RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
8259 *pGCPhysMem = GCPhys;
8260 return VINF_SUCCESS;
8261}
8262
8263
8264
8265/**
8266 * Maps a physical page.
8267 *
8268 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
8269 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8270 * @param GCPhysMem The physical address.
8271 * @param fAccess The intended access.
8272 * @param ppvMem Where to return the mapping address.
8273 * @param pLock The PGM lock.
8274 */
8275IEM_STATIC int iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
8276{
8277#ifdef IEM_LOG_MEMORY_WRITES
8278 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8279 return VERR_PGM_PHYS_TLB_CATCH_ALL;
8280#endif
8281
8282 /** @todo This API may require some improving later. A private deal with PGM
8283 * regarding locking and unlocking needs to be struct. A couple of TLBs
8284 * living in PGM, but with publicly accessible inlined access methods
8285 * could perhaps be an even better solution. */
8286 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
8287 GCPhysMem,
8288 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
8289 pVCpu->iem.s.fBypassHandlers,
8290 ppvMem,
8291 pLock);
8292 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
8293 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
8294
8295 return rc;
8296}
8297
8298
8299/**
8300 * Unmap a page previously mapped by iemMemPageMap.
8301 *
8302 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8303 * @param GCPhysMem The physical address.
8304 * @param fAccess The intended access.
8305 * @param pvMem What iemMemPageMap returned.
8306 * @param pLock The PGM lock.
8307 */
8308DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
8309{
8310 NOREF(pVCpu);
8311 NOREF(GCPhysMem);
8312 NOREF(fAccess);
8313 NOREF(pvMem);
8314 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
8315}
8316
8317
8318/**
8319 * Looks up a memory mapping entry.
8320 *
8321 * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
8322 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8323 * @param pvMem The memory address.
8324 * @param fAccess The access to.
8325 */
8326DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
8327{
8328 Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8329 fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;
8330 if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
8331 && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8332 return 0;
8333 if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
8334 && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8335 return 1;
8336 if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
8337 && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8338 return 2;
8339 return VERR_NOT_FOUND;
8340}
8341
8342
8343/**
8344 * Finds a free memmap entry when using iNextMapping doesn't work.
8345 *
8346 * @returns Memory mapping index, 1024 on failure.
8347 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8348 */
8349IEM_STATIC unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
8350{
8351 /*
8352 * The easy case.
8353 */
8354 if (pVCpu->iem.s.cActiveMappings == 0)
8355 {
8356 pVCpu->iem.s.iNextMapping = 1;
8357 return 0;
8358 }
8359
8360 /* There should be enough mappings for all instructions. */
8361 AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
8362
8363 for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
8364 if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
8365 return i;
8366
8367 AssertFailedReturn(1024);
8368}
8369
8370
8371/**
8372 * Commits a bounce buffer that needs writing back and unmaps it.
8373 *
8374 * @returns Strict VBox status code.
8375 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8376 * @param iMemMap The index of the buffer to commit.
8377 * @param fPostponeFail Whether we can postpone writer failures to ring-3.
8378 * Always false in ring-3, obviously.
8379 */
8380IEM_STATIC VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
8381{
8382 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
8383 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
8384#ifdef IN_RING3
8385 Assert(!fPostponeFail);
8386 RT_NOREF_PV(fPostponeFail);
8387#endif
8388
8389 /*
8390 * Do the writing.
8391 */
8392 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8393 if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
8394 {
8395 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
8396 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
8397 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8398 if (!pVCpu->iem.s.fBypassHandlers)
8399 {
8400 /*
8401 * Carefully and efficiently dealing with access handler return
8402 * codes make this a little bloated.
8403 */
8404 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
8405 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
8406 pbBuf,
8407 cbFirst,
8408 PGMACCESSORIGIN_IEM);
8409 if (rcStrict == VINF_SUCCESS)
8410 {
8411 if (cbSecond)
8412 {
8413 rcStrict = PGMPhysWrite(pVM,
8414 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8415 pbBuf + cbFirst,
8416 cbSecond,
8417 PGMACCESSORIGIN_IEM);
8418 if (rcStrict == VINF_SUCCESS)
8419 { /* nothing */ }
8420 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8421 {
8422 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
8423 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8424 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8425 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8426 }
8427#ifndef IN_RING3
8428 else if (fPostponeFail)
8429 {
8430 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8431 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8432 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8433 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
8434 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8435 return iemSetPassUpStatus(pVCpu, rcStrict);
8436 }
8437#endif
8438 else
8439 {
8440 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8441 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8442 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8443 return rcStrict;
8444 }
8445 }
8446 }
8447 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8448 {
8449 if (!cbSecond)
8450 {
8451 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
8452 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8453 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8454 }
8455 else
8456 {
8457 VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
8458 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8459 pbBuf + cbFirst,
8460 cbSecond,
8461 PGMACCESSORIGIN_IEM);
8462 if (rcStrict2 == VINF_SUCCESS)
8463 {
8464 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
8465 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8466 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8467 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8468 }
8469 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
8470 {
8471 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
8472 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8473 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
8474 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
8475 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8476 }
8477#ifndef IN_RING3
8478 else if (fPostponeFail)
8479 {
8480 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8481 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8482 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8483 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
8484 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8485 return iemSetPassUpStatus(pVCpu, rcStrict);
8486 }
8487#endif
8488 else
8489 {
8490 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8491 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8492 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
8493 return rcStrict2;
8494 }
8495 }
8496 }
8497#ifndef IN_RING3
8498 else if (fPostponeFail)
8499 {
8500 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8501 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8502 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8503 if (!cbSecond)
8504 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;
8505 else
8506 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;
8507 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8508 return iemSetPassUpStatus(pVCpu, rcStrict);
8509 }
8510#endif
8511 else
8512 {
8513 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
8514 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8515 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8516 return rcStrict;
8517 }
8518 }
8519 else
8520 {
8521 /*
8522 * No access handlers, much simpler.
8523 */
8524 int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
8525 if (RT_SUCCESS(rc))
8526 {
8527 if (cbSecond)
8528 {
8529 rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
8530 if (RT_SUCCESS(rc))
8531 { /* likely */ }
8532 else
8533 {
8534 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8535 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8536 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
8537 return rc;
8538 }
8539 }
8540 }
8541 else
8542 {
8543 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
8544 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
8545 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8546 return rc;
8547 }
8548 }
8549 }
8550
8551#if defined(IEM_LOG_MEMORY_WRITES)
8552 Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
8553 RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
8554 if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
8555 Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8556 RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
8557 &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
8558
8559 size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
8560 g_cbIemWrote = cbWrote;
8561 memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
8562#endif
8563
8564 /*
8565 * Free the mapping entry.
8566 */
8567 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8568 Assert(pVCpu->iem.s.cActiveMappings != 0);
8569 pVCpu->iem.s.cActiveMappings--;
8570 return VINF_SUCCESS;
8571}
8572
8573
8574/**
8575 * iemMemMap worker that deals with a request crossing pages.
8576 */
8577IEM_STATIC VBOXSTRICTRC
8578iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
8579{
8580 /*
8581 * Do the address translations.
8582 */
8583 RTGCPHYS GCPhysFirst;
8584 VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
8585 if (rcStrict != VINF_SUCCESS)
8586 return rcStrict;
8587
8588 RTGCPHYS GCPhysSecond;
8589 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
8590 fAccess, &GCPhysSecond);
8591 if (rcStrict != VINF_SUCCESS)
8592 return rcStrict;
8593 GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8594
8595 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8596
8597 /*
8598 * Read in the current memory content if it's a read, execute or partial
8599 * write access.
8600 */
8601 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8602 uint32_t const cbFirstPage = GUEST_PAGE_SIZE - (GCPhysFirst & GUEST_PAGE_OFFSET_MASK);
8603 uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
8604
8605 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
8606 {
8607 if (!pVCpu->iem.s.fBypassHandlers)
8608 {
8609 /*
8610 * Must carefully deal with access handler status codes here,
8611 * makes the code a bit bloated.
8612 */
8613 rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
8614 if (rcStrict == VINF_SUCCESS)
8615 {
8616 rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
8617 if (rcStrict == VINF_SUCCESS)
8618 { /*likely */ }
8619 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8620 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8621 else
8622 {
8623 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
8624 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8625 return rcStrict;
8626 }
8627 }
8628 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8629 {
8630 VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
8631 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
8632 {
8633 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
8634 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8635 }
8636 else
8637 {
8638 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
8639 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
8640 return rcStrict2;
8641 }
8642 }
8643 else
8644 {
8645 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8646 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8647 return rcStrict;
8648 }
8649 }
8650 else
8651 {
8652 /*
8653 * No informational status codes here, much more straight forward.
8654 */
8655 int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
8656 if (RT_SUCCESS(rc))
8657 {
8658 Assert(rc == VINF_SUCCESS);
8659 rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
8660 if (RT_SUCCESS(rc))
8661 Assert(rc == VINF_SUCCESS);
8662 else
8663 {
8664 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
8665 return rc;
8666 }
8667 }
8668 else
8669 {
8670 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
8671 return rc;
8672 }
8673 }
8674 }
8675#ifdef VBOX_STRICT
8676 else
8677 memset(pbBuf, 0xcc, cbMem);
8678 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
8679 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
8680#endif
8681
8682 /*
8683 * Commit the bounce buffer entry.
8684 */
8685 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
8686 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
8687 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
8688 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
8689 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
8690 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
8691 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
8692 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8693 pVCpu->iem.s.cActiveMappings++;
8694
8695 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8696 *ppvMem = pbBuf;
8697 return VINF_SUCCESS;
8698}
8699
8700
8701/**
8702 * iemMemMap woker that deals with iemMemPageMap failures.
8703 */
8704IEM_STATIC VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
8705 RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
8706{
8707 /*
8708 * Filter out conditions we can handle and the ones which shouldn't happen.
8709 */
8710 if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
8711 && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
8712 && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
8713 {
8714 AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
8715 return rcMap;
8716 }
8717 pVCpu->iem.s.cPotentialExits++;
8718
8719 /*
8720 * Read in the current memory content if it's a read, execute or partial
8721 * write access.
8722 */
8723 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8724 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
8725 {
8726 if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
8727 memset(pbBuf, 0xff, cbMem);
8728 else
8729 {
8730 int rc;
8731 if (!pVCpu->iem.s.fBypassHandlers)
8732 {
8733 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
8734 if (rcStrict == VINF_SUCCESS)
8735 { /* nothing */ }
8736 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8737 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8738 else
8739 {
8740 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8741 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8742 return rcStrict;
8743 }
8744 }
8745 else
8746 {
8747 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
8748 if (RT_SUCCESS(rc))
8749 { /* likely */ }
8750 else
8751 {
8752 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8753 GCPhysFirst, rc));
8754 return rc;
8755 }
8756 }
8757 }
8758 }
8759#ifdef VBOX_STRICT
8760 else
8761 memset(pbBuf, 0xcc, cbMem);
8762#endif
8763#ifdef VBOX_STRICT
8764 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
8765 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
8766#endif
8767
8768 /*
8769 * Commit the bounce buffer entry.
8770 */
8771 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
8772 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
8773 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
8774 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
8775 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
8776 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
8777 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
8778 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8779 pVCpu->iem.s.cActiveMappings++;
8780
8781 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8782 *ppvMem = pbBuf;
8783 return VINF_SUCCESS;
8784}
8785
8786
8787
8788/**
8789 * Maps the specified guest memory for the given kind of access.
8790 *
8791 * This may be using bounce buffering of the memory if it's crossing a page
8792 * boundary or if there is an access handler installed for any of it. Because
8793 * of lock prefix guarantees, we're in for some extra clutter when this
8794 * happens.
8795 *
8796 * This may raise a \#GP, \#SS, \#PF or \#AC.
8797 *
8798 * @returns VBox strict status code.
8799 *
8800 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8801 * @param ppvMem Where to return the pointer to the mapped
8802 * memory.
8803 * @param cbMem The number of bytes to map. This is usually 1,
8804 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
8805 * string operations it can be up to a page.
8806 * @param iSegReg The index of the segment register to use for
8807 * this access. The base and limits are checked.
8808 * Use UINT8_MAX to indicate that no segmentation
8809 * is required (for IDT, GDT and LDT accesses).
8810 * @param GCPtrMem The address of the guest memory.
8811 * @param fAccess How the memory is being accessed. The
8812 * IEM_ACCESS_TYPE_XXX bit is used to figure out
8813 * how to map the memory, while the
8814 * IEM_ACCESS_WHAT_XXX bit is used when raising
8815 * exceptions.
8816 */
8817IEM_STATIC VBOXSTRICTRC
8818iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
8819{
8820 /*
8821 * Check the input and figure out which mapping entry to use.
8822 */
8823 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94); /* 512 is the max! */
8824 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
8825 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8826
8827 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
8828 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
8829 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
8830 {
8831 iMemMap = iemMemMapFindFree(pVCpu);
8832 AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
8833 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
8834 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
8835 pVCpu->iem.s.aMemMappings[2].fAccess),
8836 VERR_IEM_IPE_9);
8837 }
8838
8839 /*
8840 * Map the memory, checking that we can actually access it. If something
8841 * slightly complicated happens, fall back on bounce buffering.
8842 */
8843 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
8844 if (rcStrict != VINF_SUCCESS)
8845 return rcStrict;
8846
8847 if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem > GUEST_PAGE_SIZE) /* Crossing a page boundary? */
8848 return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
8849
8850 RTGCPHYS GCPhysFirst;
8851 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
8852 if (rcStrict != VINF_SUCCESS)
8853 return rcStrict;
8854
8855 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8856 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8857 if (fAccess & IEM_ACCESS_TYPE_READ)
8858 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8859
8860 void *pvMem;
8861 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8862 if (rcStrict != VINF_SUCCESS)
8863 return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
8864
8865 /*
8866 * Fill in the mapping table entry.
8867 */
8868 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
8869 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
8870 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8871 pVCpu->iem.s.cActiveMappings++;
8872
8873 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8874 *ppvMem = pvMem;
8875
8876 return VINF_SUCCESS;
8877}
8878
8879
8880/**
8881 * Commits the guest memory if bounce buffered and unmaps it.
8882 *
8883 * @returns Strict VBox status code.
8884 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8885 * @param pvMem The mapping.
8886 * @param fAccess The kind of access.
8887 */
8888IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
8889{
8890 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
8891 AssertReturn(iMemMap >= 0, iMemMap);
8892
8893 /* If it's bounce buffered, we may need to write back the buffer. */
8894 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
8895 {
8896 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
8897 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
8898 }
8899 /* Otherwise unlock it. */
8900 else
8901 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8902
8903 /* Free the entry. */
8904 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8905 Assert(pVCpu->iem.s.cActiveMappings != 0);
8906 pVCpu->iem.s.cActiveMappings--;
8907 return VINF_SUCCESS;
8908}
8909
8910#ifdef IEM_WITH_SETJMP
8911
8912/**
8913 * Maps the specified guest memory for the given kind of access, longjmp on
8914 * error.
8915 *
8916 * This may be using bounce buffering of the memory if it's crossing a page
8917 * boundary or if there is an access handler installed for any of it. Because
8918 * of lock prefix guarantees, we're in for some extra clutter when this
8919 * happens.
8920 *
8921 * This may raise a \#GP, \#SS, \#PF or \#AC.
8922 *
8923 * @returns Pointer to the mapped memory.
8924 *
8925 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8926 * @param cbMem The number of bytes to map. This is usually 1,
8927 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
8928 * string operations it can be up to a page.
8929 * @param iSegReg The index of the segment register to use for
8930 * this access. The base and limits are checked.
8931 * Use UINT8_MAX to indicate that no segmentation
8932 * is required (for IDT, GDT and LDT accesses).
8933 * @param GCPtrMem The address of the guest memory.
8934 * @param fAccess How the memory is being accessed. The
8935 * IEM_ACCESS_TYPE_XXX bit is used to figure out
8936 * how to map the memory, while the
8937 * IEM_ACCESS_WHAT_XXX bit is used when raising
8938 * exceptions.
8939 */
8940IEM_STATIC void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
8941{
8942 /*
8943 * Check the input and figure out which mapping entry to use.
8944 */
8945 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
8946 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
8947 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8948
8949 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
8950 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
8951 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
8952 {
8953 iMemMap = iemMemMapFindFree(pVCpu);
8954 AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
8955 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
8956 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
8957 pVCpu->iem.s.aMemMappings[2].fAccess),
8958 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
8959 }
8960
8961 /*
8962 * Map the memory, checking that we can actually access it. If something
8963 * slightly complicated happens, fall back on bounce buffering.
8964 */
8965 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
8966 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
8967 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8968
8969 /* Crossing a page boundary? */
8970 if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
8971 { /* No (likely). */ }
8972 else
8973 {
8974 void *pvMem;
8975 rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
8976 if (rcStrict == VINF_SUCCESS)
8977 return pvMem;
8978 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8979 }
8980
8981 RTGCPHYS GCPhysFirst;
8982 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
8983 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
8984 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8985
8986 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8987 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8988 if (fAccess & IEM_ACCESS_TYPE_READ)
8989 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8990
8991 void *pvMem;
8992 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8993 if (rcStrict == VINF_SUCCESS)
8994 { /* likely */ }
8995 else
8996 {
8997 rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
8998 if (rcStrict == VINF_SUCCESS)
8999 return pvMem;
9000 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9001 }
9002
9003 /*
9004 * Fill in the mapping table entry.
9005 */
9006 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
9007 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
9008 pVCpu->iem.s.iNextMapping = iMemMap + 1;
9009 pVCpu->iem.s.cActiveMappings++;
9010
9011 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
9012 return pvMem;
9013}
9014
9015
9016/**
9017 * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
9018 *
9019 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9020 * @param pvMem The mapping.
9021 * @param fAccess The kind of access.
9022 */
9023IEM_STATIC void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
9024{
9025 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
9026 AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
9027
9028 /* If it's bounce buffered, we may need to write back the buffer. */
9029 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
9030 {
9031 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
9032 {
9033 VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
9034 if (rcStrict == VINF_SUCCESS)
9035 return;
9036 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9037 }
9038 }
9039 /* Otherwise unlock it. */
9040 else
9041 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9042
9043 /* Free the entry. */
9044 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9045 Assert(pVCpu->iem.s.cActiveMappings != 0);
9046 pVCpu->iem.s.cActiveMappings--;
9047}
9048
9049#endif /* IEM_WITH_SETJMP */
9050
9051#ifndef IN_RING3
9052/**
9053 * Commits the guest memory if bounce buffered and unmaps it, if any bounce
9054 * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
9055 *
9056 * Allows the instruction to be completed and retired, while the IEM user will
9057 * return to ring-3 immediately afterwards and do the postponed writes there.
9058 *
9059 * @returns VBox status code (no strict statuses). Caller must check
9060 * VMCPU_FF_IEM before repeating string instructions and similar stuff.
9061 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9062 * @param pvMem The mapping.
9063 * @param fAccess The kind of access.
9064 */
9065IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
9066{
9067 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
9068 AssertReturn(iMemMap >= 0, iMemMap);
9069
9070 /* If it's bounce buffered, we may need to write back the buffer. */
9071 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
9072 {
9073 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
9074 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);
9075 }
9076 /* Otherwise unlock it. */
9077 else
9078 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9079
9080 /* Free the entry. */
9081 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9082 Assert(pVCpu->iem.s.cActiveMappings != 0);
9083 pVCpu->iem.s.cActiveMappings--;
9084 return VINF_SUCCESS;
9085}
9086#endif
9087
9088
9089/**
9090 * Rollbacks mappings, releasing page locks and such.
9091 *
9092 * The caller shall only call this after checking cActiveMappings.
9093 *
9094 * @returns Strict VBox status code to pass up.
9095 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9096 */
9097IEM_STATIC void iemMemRollback(PVMCPUCC pVCpu)
9098{
9099 Assert(pVCpu->iem.s.cActiveMappings > 0);
9100
9101 uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
9102 while (iMemMap-- > 0)
9103 {
9104 uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
9105 if (fAccess != IEM_ACCESS_INVALID)
9106 {
9107 AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
9108 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9109 if (!(fAccess & IEM_ACCESS_BOUNCE_BUFFERED))
9110 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9111 AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
9112 ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
9113 iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
9114 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
9115 pVCpu->iem.s.cActiveMappings--;
9116 }
9117 }
9118}
9119
9120
9121/**
9122 * Fetches a data byte.
9123 *
9124 * @returns Strict VBox status code.
9125 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9126 * @param pu8Dst Where to return the byte.
9127 * @param iSegReg The index of the segment register to use for
9128 * this access. The base and limits are checked.
9129 * @param GCPtrMem The address of the guest memory.
9130 */
9131IEM_STATIC VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9132{
9133 /* The lazy approach for now... */
9134 uint8_t const *pu8Src;
9135 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9136 if (rc == VINF_SUCCESS)
9137 {
9138 *pu8Dst = *pu8Src;
9139 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
9140 }
9141 return rc;
9142}
9143
9144
9145#ifdef IEM_WITH_SETJMP
9146/**
9147 * Fetches a data byte, longjmp on error.
9148 *
9149 * @returns The byte.
9150 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9151 * @param iSegReg The index of the segment register to use for
9152 * this access. The base and limits are checked.
9153 * @param GCPtrMem The address of the guest memory.
9154 */
9155DECL_NO_INLINE(IEM_STATIC, uint8_t) iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9156{
9157 /* The lazy approach for now... */
9158 uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9159 uint8_t const bRet = *pu8Src;
9160 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
9161 return bRet;
9162}
9163#endif /* IEM_WITH_SETJMP */
9164
9165
9166/**
9167 * Fetches a data word.
9168 *
9169 * @returns Strict VBox status code.
9170 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9171 * @param pu16Dst Where to return the word.
9172 * @param iSegReg The index of the segment register to use for
9173 * this access. The base and limits are checked.
9174 * @param GCPtrMem The address of the guest memory.
9175 */
9176IEM_STATIC VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9177{
9178 /* The lazy approach for now... */
9179 uint16_t const *pu16Src;
9180 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9181 if (rc == VINF_SUCCESS)
9182 {
9183 *pu16Dst = *pu16Src;
9184 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
9185 }
9186 return rc;
9187}
9188
9189
9190#ifdef IEM_WITH_SETJMP
9191/**
9192 * Fetches a data word, longjmp on error.
9193 *
9194 * @returns The word
9195 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9196 * @param iSegReg The index of the segment register to use for
9197 * this access. The base and limits are checked.
9198 * @param GCPtrMem The address of the guest memory.
9199 */
9200DECL_NO_INLINE(IEM_STATIC, uint16_t) iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9201{
9202 /* The lazy approach for now... */
9203 uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9204 uint16_t const u16Ret = *pu16Src;
9205 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
9206 return u16Ret;
9207}
9208#endif
9209
9210
9211/**
9212 * Fetches a data dword.
9213 *
9214 * @returns Strict VBox status code.
9215 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9216 * @param pu32Dst Where to return the dword.
9217 * @param iSegReg The index of the segment register to use for
9218 * this access. The base and limits are checked.
9219 * @param GCPtrMem The address of the guest memory.
9220 */
9221IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9222{
9223 /* The lazy approach for now... */
9224 uint32_t const *pu32Src;
9225 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9226 if (rc == VINF_SUCCESS)
9227 {
9228 *pu32Dst = *pu32Src;
9229 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9230 }
9231 return rc;
9232}
9233
9234
9235/**
9236 * Fetches a data dword and zero extends it to a qword.
9237 *
9238 * @returns Strict VBox status code.
9239 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9240 * @param pu64Dst Where to return the qword.
9241 * @param iSegReg The index of the segment register to use for
9242 * this access. The base and limits are checked.
9243 * @param GCPtrMem The address of the guest memory.
9244 */
9245IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9246{
9247 /* The lazy approach for now... */
9248 uint32_t const *pu32Src;
9249 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9250 if (rc == VINF_SUCCESS)
9251 {
9252 *pu64Dst = *pu32Src;
9253 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9254 }
9255 return rc;
9256}
9257
9258
9259#ifdef IEM_WITH_SETJMP
9260
9261IEM_STATIC RTGCPTR iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
9262{
9263 Assert(cbMem >= 1);
9264 Assert(iSegReg < X86_SREG_COUNT);
9265
9266 /*
9267 * 64-bit mode is simpler.
9268 */
9269 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9270 {
9271 if (iSegReg >= X86_SREG_FS)
9272 {
9273 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9274 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9275 GCPtrMem += pSel->u64Base;
9276 }
9277
9278 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
9279 return GCPtrMem;
9280 }
9281 /*
9282 * 16-bit and 32-bit segmentation.
9283 */
9284 else
9285 {
9286 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9287 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9288 if ( (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
9289 == X86DESCATTR_P /* data, expand up */
9290 || (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ))
9291 == (X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ) /* code, read-only */ )
9292 {
9293 /* expand up */
9294 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9295 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
9296 && GCPtrLast32 > (uint32_t)GCPtrMem))
9297 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9298 }
9299 else if ( (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
9300 == (X86DESCATTR_P | X86_SEL_TYPE_DOWN) /* data, expand down */ )
9301 {
9302 /* expand down */
9303 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9304 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
9305 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
9306 && GCPtrLast32 > (uint32_t)GCPtrMem))
9307 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9308 }
9309 else
9310 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
9311 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
9312 }
9313 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
9314}
9315
9316
9317IEM_STATIC RTGCPTR iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
9318{
9319 Assert(cbMem >= 1);
9320 Assert(iSegReg < X86_SREG_COUNT);
9321
9322 /*
9323 * 64-bit mode is simpler.
9324 */
9325 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9326 {
9327 if (iSegReg >= X86_SREG_FS)
9328 {
9329 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9330 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9331 GCPtrMem += pSel->u64Base;
9332 }
9333
9334 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
9335 return GCPtrMem;
9336 }
9337 /*
9338 * 16-bit and 32-bit segmentation.
9339 */
9340 else
9341 {
9342 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9343 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9344 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
9345 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
9346 if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */
9347 {
9348 /* expand up */
9349 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9350 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
9351 && GCPtrLast32 > (uint32_t)GCPtrMem))
9352 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9353 }
9354 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
9355 {
9356 /* expand down */
9357 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9358 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
9359 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
9360 && GCPtrLast32 > (uint32_t)GCPtrMem))
9361 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9362 }
9363 else
9364 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
9365 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
9366 }
9367 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
9368}
9369
9370
9371/**
9372 * Fetches a data dword, longjmp on error, fallback/safe version.
9373 *
9374 * @returns The dword
9375 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9376 * @param iSegReg The index of the segment register to use for
9377 * this access. The base and limits are checked.
9378 * @param GCPtrMem The address of the guest memory.
9379 */
9380IEM_STATIC uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9381{
9382 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9383 uint32_t const u32Ret = *pu32Src;
9384 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9385 return u32Ret;
9386}
9387
9388
9389/**
9390 * Fetches a data dword, longjmp on error.
9391 *
9392 * @returns The dword
9393 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9394 * @param iSegReg The index of the segment register to use for
9395 * this access. The base and limits are checked.
9396 * @param GCPtrMem The address of the guest memory.
9397 */
9398DECL_NO_INLINE(IEM_STATIC, uint32_t) iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9399{
9400# ifdef IEM_WITH_DATA_TLB
9401 RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
9402 if (RT_LIKELY((GCPtrEff & X86_PAGE_OFFSET_MASK) <= X86_PAGE_SIZE - sizeof(uint32_t)))
9403 {
9404 /// @todo more later.
9405 }
9406
9407 return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
9408# else
9409 /* The lazy approach. */
9410 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9411 uint32_t const u32Ret = *pu32Src;
9412 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9413 return u32Ret;
9414# endif
9415}
9416#endif
9417
9418
9419#ifdef SOME_UNUSED_FUNCTION
9420/**
9421 * Fetches a data dword and sign extends it to a qword.
9422 *
9423 * @returns Strict VBox status code.
9424 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9425 * @param pu64Dst Where to return the sign extended value.
9426 * @param iSegReg The index of the segment register to use for
9427 * this access. The base and limits are checked.
9428 * @param GCPtrMem The address of the guest memory.
9429 */
9430IEM_STATIC VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9431{
9432 /* The lazy approach for now... */
9433 int32_t const *pi32Src;
9434 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9435 if (rc == VINF_SUCCESS)
9436 {
9437 *pu64Dst = *pi32Src;
9438 rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
9439 }
9440#ifdef __GNUC__ /* warning: GCC may be a royal pain */
9441 else
9442 *pu64Dst = 0;
9443#endif
9444 return rc;
9445}
9446#endif
9447
9448
9449/**
9450 * Fetches a data qword.
9451 *
9452 * @returns Strict VBox status code.
9453 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9454 * @param pu64Dst Where to return the qword.
9455 * @param iSegReg The index of the segment register to use for
9456 * this access. The base and limits are checked.
9457 * @param GCPtrMem The address of the guest memory.
9458 */
9459IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9460{
9461 /* The lazy approach for now... */
9462 uint64_t const *pu64Src;
9463 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9464 if (rc == VINF_SUCCESS)
9465 {
9466 *pu64Dst = *pu64Src;
9467 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9468 }
9469 return rc;
9470}
9471
9472
9473#ifdef IEM_WITH_SETJMP
9474/**
9475 * Fetches a data qword, longjmp on error.
9476 *
9477 * @returns The qword.
9478 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9479 * @param iSegReg The index of the segment register to use for
9480 * this access. The base and limits are checked.
9481 * @param GCPtrMem The address of the guest memory.
9482 */
9483DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9484{
9485 /* The lazy approach for now... */
9486 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9487 uint64_t const u64Ret = *pu64Src;
9488 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9489 return u64Ret;
9490}
9491#endif
9492
9493
9494/**
9495 * Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
9496 *
9497 * @returns Strict VBox status code.
9498 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9499 * @param pu64Dst Where to return the qword.
9500 * @param iSegReg The index of the segment register to use for
9501 * this access. The base and limits are checked.
9502 * @param GCPtrMem The address of the guest memory.
9503 */
9504IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9505{
9506 /* The lazy approach for now... */
9507 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9508 if (RT_UNLIKELY(GCPtrMem & 15))
9509 return iemRaiseGeneralProtectionFault0(pVCpu);
9510
9511 uint64_t const *pu64Src;
9512 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9513 if (rc == VINF_SUCCESS)
9514 {
9515 *pu64Dst = *pu64Src;
9516 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9517 }
9518 return rc;
9519}
9520
9521
9522#ifdef IEM_WITH_SETJMP
9523/**
9524 * Fetches a data qword, longjmp on error.
9525 *
9526 * @returns The qword.
9527 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9528 * @param iSegReg The index of the segment register to use for
9529 * this access. The base and limits are checked.
9530 * @param GCPtrMem The address of the guest memory.
9531 */
9532DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9533{
9534 /* The lazy approach for now... */
9535 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9536 if (RT_LIKELY(!(GCPtrMem & 15)))
9537 {
9538 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9539 uint64_t const u64Ret = *pu64Src;
9540 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9541 return u64Ret;
9542 }
9543
9544 VBOXSTRICTRC rc = iemRaiseGeneralProtectionFault0(pVCpu);
9545 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rc));
9546}
9547#endif
9548
9549
9550/**
9551 * Fetches a data tword.
9552 *
9553 * @returns Strict VBox status code.
9554 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9555 * @param pr80Dst Where to return the tword.
9556 * @param iSegReg The index of the segment register to use for
9557 * this access. The base and limits are checked.
9558 * @param GCPtrMem The address of the guest memory.
9559 */
9560IEM_STATIC VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9561{
9562 /* The lazy approach for now... */
9563 PCRTFLOAT80U pr80Src;
9564 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9565 if (rc == VINF_SUCCESS)
9566 {
9567 *pr80Dst = *pr80Src;
9568 rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
9569 }
9570 return rc;
9571}
9572
9573
9574#ifdef IEM_WITH_SETJMP
9575/**
9576 * Fetches a data tword, longjmp on error.
9577 *
9578 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9579 * @param pr80Dst Where to return the tword.
9580 * @param iSegReg The index of the segment register to use for
9581 * this access. The base and limits are checked.
9582 * @param GCPtrMem The address of the guest memory.
9583 */
9584DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9585{
9586 /* The lazy approach for now... */
9587 PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9588 *pr80Dst = *pr80Src;
9589 iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
9590}
9591#endif
9592
9593
9594/**
9595 * Fetches a data dqword (double qword), generally SSE related.
9596 *
9597 * @returns Strict VBox status code.
9598 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9599 * @param pu128Dst Where to return the qword.
9600 * @param iSegReg The index of the segment register to use for
9601 * this access. The base and limits are checked.
9602 * @param GCPtrMem The address of the guest memory.
9603 */
9604IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9605{
9606 /* The lazy approach for now... */
9607 PCRTUINT128U pu128Src;
9608 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9609 if (rc == VINF_SUCCESS)
9610 {
9611 pu128Dst->au64[0] = pu128Src->au64[0];
9612 pu128Dst->au64[1] = pu128Src->au64[1];
9613 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9614 }
9615 return rc;
9616}
9617
9618
9619#ifdef IEM_WITH_SETJMP
9620/**
9621 * Fetches a data dqword (double qword), generally SSE related.
9622 *
9623 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9624 * @param pu128Dst Where to return the qword.
9625 * @param iSegReg The index of the segment register to use for
9626 * this access. The base and limits are checked.
9627 * @param GCPtrMem The address of the guest memory.
9628 */
9629IEM_STATIC void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9630{
9631 /* The lazy approach for now... */
9632 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9633 pu128Dst->au64[0] = pu128Src->au64[0];
9634 pu128Dst->au64[1] = pu128Src->au64[1];
9635 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9636}
9637#endif
9638
9639
9640/**
9641 * Fetches a data dqword (double qword) at an aligned address, generally SSE
9642 * related.
9643 *
9644 * Raises \#GP(0) if not aligned.
9645 *
9646 * @returns Strict VBox status code.
9647 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9648 * @param pu128Dst Where to return the qword.
9649 * @param iSegReg The index of the segment register to use for
9650 * this access. The base and limits are checked.
9651 * @param GCPtrMem The address of the guest memory.
9652 */
9653IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9654{
9655 /* The lazy approach for now... */
9656 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9657 if ( (GCPtrMem & 15)
9658 && !(pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
9659 return iemRaiseGeneralProtectionFault0(pVCpu);
9660
9661 PCRTUINT128U pu128Src;
9662 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9663 if (rc == VINF_SUCCESS)
9664 {
9665 pu128Dst->au64[0] = pu128Src->au64[0];
9666 pu128Dst->au64[1] = pu128Src->au64[1];
9667 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9668 }
9669 return rc;
9670}
9671
9672
9673#ifdef IEM_WITH_SETJMP
9674/**
9675 * Fetches a data dqword (double qword) at an aligned address, generally SSE
9676 * related, longjmp on error.
9677 *
9678 * Raises \#GP(0) if not aligned.
9679 *
9680 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9681 * @param pu128Dst Where to return the qword.
9682 * @param iSegReg The index of the segment register to use for
9683 * this access. The base and limits are checked.
9684 * @param GCPtrMem The address of the guest memory.
9685 */
9686DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9687{
9688 /* The lazy approach for now... */
9689 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9690 if ( (GCPtrMem & 15) == 0
9691 || (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
9692 {
9693 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9694 pu128Dst->au64[0] = pu128Src->au64[0];
9695 pu128Dst->au64[1] = pu128Src->au64[1];
9696 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9697 return;
9698 }
9699
9700 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
9701 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9702}
9703#endif
9704
9705
9706/**
9707 * Fetches a data oword (octo word), generally AVX related.
9708 *
9709 * @returns Strict VBox status code.
9710 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9711 * @param pu256Dst Where to return the qword.
9712 * @param iSegReg The index of the segment register to use for
9713 * this access. The base and limits are checked.
9714 * @param GCPtrMem The address of the guest memory.
9715 */
9716IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9717{
9718 /* The lazy approach for now... */
9719 PCRTUINT256U pu256Src;
9720 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9721 if (rc == VINF_SUCCESS)
9722 {
9723 pu256Dst->au64[0] = pu256Src->au64[0];
9724 pu256Dst->au64[1] = pu256Src->au64[1];
9725 pu256Dst->au64[2] = pu256Src->au64[2];
9726 pu256Dst->au64[3] = pu256Src->au64[3];
9727 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9728 }
9729 return rc;
9730}
9731
9732
9733#ifdef IEM_WITH_SETJMP
9734/**
9735 * Fetches a data oword (octo word), generally AVX related.
9736 *
9737 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9738 * @param pu256Dst Where to return the qword.
9739 * @param iSegReg The index of the segment register to use for
9740 * this access. The base and limits are checked.
9741 * @param GCPtrMem The address of the guest memory.
9742 */
9743IEM_STATIC void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9744{
9745 /* The lazy approach for now... */
9746 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9747 pu256Dst->au64[0] = pu256Src->au64[0];
9748 pu256Dst->au64[1] = pu256Src->au64[1];
9749 pu256Dst->au64[2] = pu256Src->au64[2];
9750 pu256Dst->au64[3] = pu256Src->au64[3];
9751 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9752}
9753#endif
9754
9755
9756/**
9757 * Fetches a data oword (octo word) at an aligned address, generally AVX
9758 * related.
9759 *
9760 * Raises \#GP(0) if not aligned.
9761 *
9762 * @returns Strict VBox status code.
9763 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9764 * @param pu256Dst Where to return the qword.
9765 * @param iSegReg The index of the segment register to use for
9766 * this access. The base and limits are checked.
9767 * @param GCPtrMem The address of the guest memory.
9768 */
9769IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9770{
9771 /* The lazy approach for now... */
9772 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
9773 if (GCPtrMem & 31)
9774 return iemRaiseGeneralProtectionFault0(pVCpu);
9775
9776 PCRTUINT256U pu256Src;
9777 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9778 if (rc == VINF_SUCCESS)
9779 {
9780 pu256Dst->au64[0] = pu256Src->au64[0];
9781 pu256Dst->au64[1] = pu256Src->au64[1];
9782 pu256Dst->au64[2] = pu256Src->au64[2];
9783 pu256Dst->au64[3] = pu256Src->au64[3];
9784 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9785 }
9786 return rc;
9787}
9788
9789
9790#ifdef IEM_WITH_SETJMP
9791/**
9792 * Fetches a data oword (octo word) at an aligned address, generally AVX
9793 * related, longjmp on error.
9794 *
9795 * Raises \#GP(0) if not aligned.
9796 *
9797 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9798 * @param pu256Dst Where to return the qword.
9799 * @param iSegReg The index of the segment register to use for
9800 * this access. The base and limits are checked.
9801 * @param GCPtrMem The address of the guest memory.
9802 */
9803DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9804{
9805 /* The lazy approach for now... */
9806 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
9807 if ((GCPtrMem & 31) == 0)
9808 {
9809 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9810 pu256Dst->au64[0] = pu256Src->au64[0];
9811 pu256Dst->au64[1] = pu256Src->au64[1];
9812 pu256Dst->au64[2] = pu256Src->au64[2];
9813 pu256Dst->au64[3] = pu256Src->au64[3];
9814 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9815 return;
9816 }
9817
9818 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
9819 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9820}
9821#endif
9822
9823
9824
9825/**
9826 * Fetches a descriptor register (lgdt, lidt).
9827 *
9828 * @returns Strict VBox status code.
9829 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9830 * @param pcbLimit Where to return the limit.
9831 * @param pGCPtrBase Where to return the base.
9832 * @param iSegReg The index of the segment register to use for
9833 * this access. The base and limits are checked.
9834 * @param GCPtrMem The address of the guest memory.
9835 * @param enmOpSize The effective operand size.
9836 */
9837IEM_STATIC VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
9838 RTGCPTR GCPtrMem, IEMMODE enmOpSize)
9839{
9840 /*
9841 * Just like SIDT and SGDT, the LIDT and LGDT instructions are a
9842 * little special:
9843 * - The two reads are done separately.
9844 * - Operand size override works in 16-bit and 32-bit code, but 64-bit.
9845 * - We suspect the 386 to actually commit the limit before the base in
9846 * some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
9847 * don't try emulate this eccentric behavior, because it's not well
9848 * enough understood and rather hard to trigger.
9849 * - The 486 seems to do a dword limit read when the operand size is 32-bit.
9850 */
9851 VBOXSTRICTRC rcStrict;
9852 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9853 {
9854 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9855 if (rcStrict == VINF_SUCCESS)
9856 rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
9857 }
9858 else
9859 {
9860 uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
9861 if (enmOpSize == IEMMODE_32BIT)
9862 {
9863 if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
9864 {
9865 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9866 if (rcStrict == VINF_SUCCESS)
9867 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9868 }
9869 else
9870 {
9871 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
9872 if (rcStrict == VINF_SUCCESS)
9873 {
9874 *pcbLimit = (uint16_t)uTmp;
9875 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9876 }
9877 }
9878 if (rcStrict == VINF_SUCCESS)
9879 *pGCPtrBase = uTmp;
9880 }
9881 else
9882 {
9883 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9884 if (rcStrict == VINF_SUCCESS)
9885 {
9886 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9887 if (rcStrict == VINF_SUCCESS)
9888 *pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
9889 }
9890 }
9891 }
9892 return rcStrict;
9893}
9894
9895
9896
9897/**
9898 * Stores a data byte.
9899 *
9900 * @returns Strict VBox status code.
9901 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9902 * @param iSegReg The index of the segment register to use for
9903 * this access. The base and limits are checked.
9904 * @param GCPtrMem The address of the guest memory.
9905 * @param u8Value The value to store.
9906 */
9907IEM_STATIC VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
9908{
9909 /* The lazy approach for now... */
9910 uint8_t *pu8Dst;
9911 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9912 if (rc == VINF_SUCCESS)
9913 {
9914 *pu8Dst = u8Value;
9915 rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
9916 }
9917 return rc;
9918}
9919
9920
9921#ifdef IEM_WITH_SETJMP
9922/**
9923 * Stores a data byte, longjmp on error.
9924 *
9925 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9926 * @param iSegReg The index of the segment register to use for
9927 * this access. The base and limits are checked.
9928 * @param GCPtrMem The address of the guest memory.
9929 * @param u8Value The value to store.
9930 */
9931IEM_STATIC void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
9932{
9933 /* The lazy approach for now... */
9934 uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9935 *pu8Dst = u8Value;
9936 iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
9937}
9938#endif
9939
9940
9941/**
9942 * Stores a data word.
9943 *
9944 * @returns Strict VBox status code.
9945 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9946 * @param iSegReg The index of the segment register to use for
9947 * this access. The base and limits are checked.
9948 * @param GCPtrMem The address of the guest memory.
9949 * @param u16Value The value to store.
9950 */
9951IEM_STATIC VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
9952{
9953 /* The lazy approach for now... */
9954 uint16_t *pu16Dst;
9955 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9956 if (rc == VINF_SUCCESS)
9957 {
9958 *pu16Dst = u16Value;
9959 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
9960 }
9961 return rc;
9962}
9963
9964
9965#ifdef IEM_WITH_SETJMP
9966/**
9967 * Stores a data word, longjmp on error.
9968 *
9969 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9970 * @param iSegReg The index of the segment register to use for
9971 * this access. The base and limits are checked.
9972 * @param GCPtrMem The address of the guest memory.
9973 * @param u16Value The value to store.
9974 */
9975IEM_STATIC void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
9976{
9977 /* The lazy approach for now... */
9978 uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9979 *pu16Dst = u16Value;
9980 iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
9981}
9982#endif
9983
9984
9985/**
9986 * Stores a data dword.
9987 *
9988 * @returns Strict VBox status code.
9989 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9990 * @param iSegReg The index of the segment register to use for
9991 * this access. The base and limits are checked.
9992 * @param GCPtrMem The address of the guest memory.
9993 * @param u32Value The value to store.
9994 */
9995IEM_STATIC VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
9996{
9997 /* The lazy approach for now... */
9998 uint32_t *pu32Dst;
9999 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10000 if (rc == VINF_SUCCESS)
10001 {
10002 *pu32Dst = u32Value;
10003 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
10004 }
10005 return rc;
10006}
10007
10008
10009#ifdef IEM_WITH_SETJMP
10010/**
10011 * Stores a data dword.
10012 *
10013 * @returns Strict VBox status code.
10014 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10015 * @param iSegReg The index of the segment register to use for
10016 * this access. The base and limits are checked.
10017 * @param GCPtrMem The address of the guest memory.
10018 * @param u32Value The value to store.
10019 */
10020IEM_STATIC void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
10021{
10022 /* The lazy approach for now... */
10023 uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10024 *pu32Dst = u32Value;
10025 iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
10026}
10027#endif
10028
10029
10030/**
10031 * Stores a data qword.
10032 *
10033 * @returns Strict VBox status code.
10034 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10035 * @param iSegReg The index of the segment register to use for
10036 * this access. The base and limits are checked.
10037 * @param GCPtrMem The address of the guest memory.
10038 * @param u64Value The value to store.
10039 */
10040IEM_STATIC VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
10041{
10042 /* The lazy approach for now... */
10043 uint64_t *pu64Dst;
10044 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10045 if (rc == VINF_SUCCESS)
10046 {
10047 *pu64Dst = u64Value;
10048 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
10049 }
10050 return rc;
10051}
10052
10053
10054#ifdef IEM_WITH_SETJMP
10055/**
10056 * Stores a data qword, longjmp on error.
10057 *
10058 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10059 * @param iSegReg The index of the segment register to use for
10060 * this access. The base and limits are checked.
10061 * @param GCPtrMem The address of the guest memory.
10062 * @param u64Value The value to store.
10063 */
10064IEM_STATIC void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
10065{
10066 /* The lazy approach for now... */
10067 uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10068 *pu64Dst = u64Value;
10069 iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
10070}
10071#endif
10072
10073
10074/**
10075 * Stores a data dqword.
10076 *
10077 * @returns Strict VBox status code.
10078 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10079 * @param iSegReg The index of the segment register to use for
10080 * this access. The base and limits are checked.
10081 * @param GCPtrMem The address of the guest memory.
10082 * @param u128Value The value to store.
10083 */
10084IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10085{
10086 /* The lazy approach for now... */
10087 PRTUINT128U pu128Dst;
10088 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10089 if (rc == VINF_SUCCESS)
10090 {
10091 pu128Dst->au64[0] = u128Value.au64[0];
10092 pu128Dst->au64[1] = u128Value.au64[1];
10093 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10094 }
10095 return rc;
10096}
10097
10098
10099#ifdef IEM_WITH_SETJMP
10100/**
10101 * Stores a data dqword, longjmp on error.
10102 *
10103 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10104 * @param iSegReg The index of the segment register to use for
10105 * this access. The base and limits are checked.
10106 * @param GCPtrMem The address of the guest memory.
10107 * @param u128Value The value to store.
10108 */
10109IEM_STATIC void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10110{
10111 /* The lazy approach for now... */
10112 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10113 pu128Dst->au64[0] = u128Value.au64[0];
10114 pu128Dst->au64[1] = u128Value.au64[1];
10115 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10116}
10117#endif
10118
10119
10120/**
10121 * Stores a data dqword, SSE aligned.
10122 *
10123 * @returns Strict VBox status code.
10124 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10125 * @param iSegReg The index of the segment register to use for
10126 * this access. The base and limits are checked.
10127 * @param GCPtrMem The address of the guest memory.
10128 * @param u128Value The value to store.
10129 */
10130IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10131{
10132 /* The lazy approach for now... */
10133 if ( (GCPtrMem & 15)
10134 && !(pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
10135 return iemRaiseGeneralProtectionFault0(pVCpu);
10136
10137 PRTUINT128U pu128Dst;
10138 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10139 if (rc == VINF_SUCCESS)
10140 {
10141 pu128Dst->au64[0] = u128Value.au64[0];
10142 pu128Dst->au64[1] = u128Value.au64[1];
10143 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10144 }
10145 return rc;
10146}
10147
10148
10149#ifdef IEM_WITH_SETJMP
10150/**
10151 * Stores a data dqword, SSE aligned.
10152 *
10153 * @returns Strict VBox status code.
10154 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10155 * @param iSegReg The index of the segment register to use for
10156 * this access. The base and limits are checked.
10157 * @param GCPtrMem The address of the guest memory.
10158 * @param u128Value The value to store.
10159 */
10160DECL_NO_INLINE(IEM_STATIC, void)
10161iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10162{
10163 /* The lazy approach for now... */
10164 if ( (GCPtrMem & 15) == 0
10165 || (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
10166 {
10167 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10168 pu128Dst->au64[0] = u128Value.au64[0];
10169 pu128Dst->au64[1] = u128Value.au64[1];
10170 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10171 return;
10172 }
10173
10174 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
10175 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
10176}
10177#endif
10178
10179
10180/**
10181 * Stores a data dqword.
10182 *
10183 * @returns Strict VBox status code.
10184 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10185 * @param iSegReg The index of the segment register to use for
10186 * this access. The base and limits are checked.
10187 * @param GCPtrMem The address of the guest memory.
10188 * @param pu256Value Pointer to the value to store.
10189 */
10190IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10191{
10192 /* The lazy approach for now... */
10193 PRTUINT256U pu256Dst;
10194 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10195 if (rc == VINF_SUCCESS)
10196 {
10197 pu256Dst->au64[0] = pu256Value->au64[0];
10198 pu256Dst->au64[1] = pu256Value->au64[1];
10199 pu256Dst->au64[2] = pu256Value->au64[2];
10200 pu256Dst->au64[3] = pu256Value->au64[3];
10201 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10202 }
10203 return rc;
10204}
10205
10206
10207#ifdef IEM_WITH_SETJMP
10208/**
10209 * Stores a data dqword, longjmp on error.
10210 *
10211 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10212 * @param iSegReg The index of the segment register to use for
10213 * this access. The base and limits are checked.
10214 * @param GCPtrMem The address of the guest memory.
10215 * @param pu256Value Pointer to the value to store.
10216 */
10217IEM_STATIC void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10218{
10219 /* The lazy approach for now... */
10220 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10221 pu256Dst->au64[0] = pu256Value->au64[0];
10222 pu256Dst->au64[1] = pu256Value->au64[1];
10223 pu256Dst->au64[2] = pu256Value->au64[2];
10224 pu256Dst->au64[3] = pu256Value->au64[3];
10225 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10226}
10227#endif
10228
10229
10230/**
10231 * Stores a data dqword, AVX aligned.
10232 *
10233 * @returns Strict VBox status code.
10234 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10235 * @param iSegReg The index of the segment register to use for
10236 * this access. The base and limits are checked.
10237 * @param GCPtrMem The address of the guest memory.
10238 * @param pu256Value Pointer to the value to store.
10239 */
10240IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10241{
10242 /* The lazy approach for now... */
10243 if (GCPtrMem & 31)
10244 return iemRaiseGeneralProtectionFault0(pVCpu);
10245
10246 PRTUINT256U pu256Dst;
10247 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10248 if (rc == VINF_SUCCESS)
10249 {
10250 pu256Dst->au64[0] = pu256Value->au64[0];
10251 pu256Dst->au64[1] = pu256Value->au64[1];
10252 pu256Dst->au64[2] = pu256Value->au64[2];
10253 pu256Dst->au64[3] = pu256Value->au64[3];
10254 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10255 }
10256 return rc;
10257}
10258
10259
10260#ifdef IEM_WITH_SETJMP
10261/**
10262 * Stores a data dqword, AVX aligned.
10263 *
10264 * @returns Strict VBox status code.
10265 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10266 * @param iSegReg The index of the segment register to use for
10267 * this access. The base and limits are checked.
10268 * @param GCPtrMem The address of the guest memory.
10269 * @param pu256Value Pointer to the value to store.
10270 */
10271DECL_NO_INLINE(IEM_STATIC, void)
10272iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10273{
10274 /* The lazy approach for now... */
10275 if ((GCPtrMem & 31) == 0)
10276 {
10277 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10278 pu256Dst->au64[0] = pu256Value->au64[0];
10279 pu256Dst->au64[1] = pu256Value->au64[1];
10280 pu256Dst->au64[2] = pu256Value->au64[2];
10281 pu256Dst->au64[3] = pu256Value->au64[3];
10282 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10283 return;
10284 }
10285
10286 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
10287 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
10288}
10289#endif
10290
10291
10292/**
10293 * Stores a descriptor register (sgdt, sidt).
10294 *
10295 * @returns Strict VBox status code.
10296 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10297 * @param cbLimit The limit.
10298 * @param GCPtrBase The base address.
10299 * @param iSegReg The index of the segment register to use for
10300 * this access. The base and limits are checked.
10301 * @param GCPtrMem The address of the guest memory.
10302 */
10303IEM_STATIC VBOXSTRICTRC
10304iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem)
10305{
10306 /*
10307 * The SIDT and SGDT instructions actually stores the data using two
10308 * independent writes. The instructions does not respond to opsize prefixes.
10309 */
10310 VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
10311 if (rcStrict == VINF_SUCCESS)
10312 {
10313 if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
10314 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
10315 IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
10316 ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
10317 else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
10318 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
10319 else
10320 rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
10321 }
10322 return rcStrict;
10323}
10324
10325
10326/**
10327 * Pushes a word onto the stack.
10328 *
10329 * @returns Strict VBox status code.
10330 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10331 * @param u16Value The value to push.
10332 */
10333IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value)
10334{
10335 /* Increment the stack pointer. */
10336 uint64_t uNewRsp;
10337 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
10338
10339 /* Write the word the lazy way. */
10340 uint16_t *pu16Dst;
10341 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10342 if (rc == VINF_SUCCESS)
10343 {
10344 *pu16Dst = u16Value;
10345 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
10346 }
10347
10348 /* Commit the new RSP value unless we an access handler made trouble. */
10349 if (rc == VINF_SUCCESS)
10350 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10351
10352 return rc;
10353}
10354
10355
10356/**
10357 * Pushes a dword onto the stack.
10358 *
10359 * @returns Strict VBox status code.
10360 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10361 * @param u32Value The value to push.
10362 */
10363IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value)
10364{
10365 /* Increment the stack pointer. */
10366 uint64_t uNewRsp;
10367 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
10368
10369 /* Write the dword the lazy way. */
10370 uint32_t *pu32Dst;
10371 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10372 if (rc == VINF_SUCCESS)
10373 {
10374 *pu32Dst = u32Value;
10375 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
10376 }
10377
10378 /* Commit the new RSP value unless we an access handler made trouble. */
10379 if (rc == VINF_SUCCESS)
10380 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10381
10382 return rc;
10383}
10384
10385
10386/**
10387 * Pushes a dword segment register value onto the stack.
10388 *
10389 * @returns Strict VBox status code.
10390 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10391 * @param u32Value The value to push.
10392 */
10393IEM_STATIC VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value)
10394{
10395 /* Increment the stack pointer. */
10396 uint64_t uNewRsp;
10397 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
10398
10399 /* The intel docs talks about zero extending the selector register
10400 value. My actual intel CPU here might be zero extending the value
10401 but it still only writes the lower word... */
10402 /** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
10403 * happens when crossing an electric page boundrary, is the high word checked
10404 * for write accessibility or not? Probably it is. What about segment limits?
10405 * It appears this behavior is also shared with trap error codes.
10406 *
10407 * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
10408 * ancient hardware when it actually did change. */
10409 uint16_t *pu16Dst;
10410 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_RW);
10411 if (rc == VINF_SUCCESS)
10412 {
10413 *pu16Dst = (uint16_t)u32Value;
10414 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
10415 }
10416
10417 /* Commit the new RSP value unless we an access handler made trouble. */
10418 if (rc == VINF_SUCCESS)
10419 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10420
10421 return rc;
10422}
10423
10424
10425/**
10426 * Pushes a qword onto the stack.
10427 *
10428 * @returns Strict VBox status code.
10429 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10430 * @param u64Value The value to push.
10431 */
10432IEM_STATIC VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value)
10433{
10434 /* Increment the stack pointer. */
10435 uint64_t uNewRsp;
10436 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
10437
10438 /* Write the word the lazy way. */
10439 uint64_t *pu64Dst;
10440 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10441 if (rc == VINF_SUCCESS)
10442 {
10443 *pu64Dst = u64Value;
10444 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
10445 }
10446
10447 /* Commit the new RSP value unless we an access handler made trouble. */
10448 if (rc == VINF_SUCCESS)
10449 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10450
10451 return rc;
10452}
10453
10454
10455/**
10456 * Pops a word from the stack.
10457 *
10458 * @returns Strict VBox status code.
10459 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10460 * @param pu16Value Where to store the popped value.
10461 */
10462IEM_STATIC VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value)
10463{
10464 /* Increment the stack pointer. */
10465 uint64_t uNewRsp;
10466 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
10467
10468 /* Write the word the lazy way. */
10469 uint16_t const *pu16Src;
10470 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10471 if (rc == VINF_SUCCESS)
10472 {
10473 *pu16Value = *pu16Src;
10474 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
10475
10476 /* Commit the new RSP value. */
10477 if (rc == VINF_SUCCESS)
10478 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10479 }
10480
10481 return rc;
10482}
10483
10484
10485/**
10486 * Pops a dword from the stack.
10487 *
10488 * @returns Strict VBox status code.
10489 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10490 * @param pu32Value Where to store the popped value.
10491 */
10492IEM_STATIC VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value)
10493{
10494 /* Increment the stack pointer. */
10495 uint64_t uNewRsp;
10496 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
10497
10498 /* Write the word the lazy way. */
10499 uint32_t const *pu32Src;
10500 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10501 if (rc == VINF_SUCCESS)
10502 {
10503 *pu32Value = *pu32Src;
10504 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
10505
10506 /* Commit the new RSP value. */
10507 if (rc == VINF_SUCCESS)
10508 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10509 }
10510
10511 return rc;
10512}
10513
10514
10515/**
10516 * Pops a qword from the stack.
10517 *
10518 * @returns Strict VBox status code.
10519 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10520 * @param pu64Value Where to store the popped value.
10521 */
10522IEM_STATIC VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value)
10523{
10524 /* Increment the stack pointer. */
10525 uint64_t uNewRsp;
10526 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
10527
10528 /* Write the word the lazy way. */
10529 uint64_t const *pu64Src;
10530 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10531 if (rc == VINF_SUCCESS)
10532 {
10533 *pu64Value = *pu64Src;
10534 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
10535
10536 /* Commit the new RSP value. */
10537 if (rc == VINF_SUCCESS)
10538 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10539 }
10540
10541 return rc;
10542}
10543
10544
10545/**
10546 * Pushes a word onto the stack, using a temporary stack pointer.
10547 *
10548 * @returns Strict VBox status code.
10549 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10550 * @param u16Value The value to push.
10551 * @param pTmpRsp Pointer to the temporary stack pointer.
10552 */
10553IEM_STATIC VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
10554{
10555 /* Increment the stack pointer. */
10556 RTUINT64U NewRsp = *pTmpRsp;
10557 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
10558
10559 /* Write the word the lazy way. */
10560 uint16_t *pu16Dst;
10561 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10562 if (rc == VINF_SUCCESS)
10563 {
10564 *pu16Dst = u16Value;
10565 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
10566 }
10567
10568 /* Commit the new RSP value unless we an access handler made trouble. */
10569 if (rc == VINF_SUCCESS)
10570 *pTmpRsp = NewRsp;
10571
10572 return rc;
10573}
10574
10575
10576/**
10577 * Pushes a dword onto the stack, using a temporary stack pointer.
10578 *
10579 * @returns Strict VBox status code.
10580 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10581 * @param u32Value The value to push.
10582 * @param pTmpRsp Pointer to the temporary stack pointer.
10583 */
10584IEM_STATIC VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
10585{
10586 /* Increment the stack pointer. */
10587 RTUINT64U NewRsp = *pTmpRsp;
10588 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
10589
10590 /* Write the word the lazy way. */
10591 uint32_t *pu32Dst;
10592 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10593 if (rc == VINF_SUCCESS)
10594 {
10595 *pu32Dst = u32Value;
10596 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
10597 }
10598
10599 /* Commit the new RSP value unless we an access handler made trouble. */
10600 if (rc == VINF_SUCCESS)
10601 *pTmpRsp = NewRsp;
10602
10603 return rc;
10604}
10605
10606
10607/**
10608 * Pushes a dword onto the stack, using a temporary stack pointer.
10609 *
10610 * @returns Strict VBox status code.
10611 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10612 * @param u64Value The value to push.
10613 * @param pTmpRsp Pointer to the temporary stack pointer.
10614 */
10615IEM_STATIC VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
10616{
10617 /* Increment the stack pointer. */
10618 RTUINT64U NewRsp = *pTmpRsp;
10619 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
10620
10621 /* Write the word the lazy way. */
10622 uint64_t *pu64Dst;
10623 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10624 if (rc == VINF_SUCCESS)
10625 {
10626 *pu64Dst = u64Value;
10627 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
10628 }
10629
10630 /* Commit the new RSP value unless we an access handler made trouble. */
10631 if (rc == VINF_SUCCESS)
10632 *pTmpRsp = NewRsp;
10633
10634 return rc;
10635}
10636
10637
10638/**
10639 * Pops a word from the stack, using a temporary stack pointer.
10640 *
10641 * @returns Strict VBox status code.
10642 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10643 * @param pu16Value Where to store the popped value.
10644 * @param pTmpRsp Pointer to the temporary stack pointer.
10645 */
10646IEM_STATIC VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
10647{
10648 /* Increment the stack pointer. */
10649 RTUINT64U NewRsp = *pTmpRsp;
10650 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
10651
10652 /* Write the word the lazy way. */
10653 uint16_t const *pu16Src;
10654 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10655 if (rc == VINF_SUCCESS)
10656 {
10657 *pu16Value = *pu16Src;
10658 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
10659
10660 /* Commit the new RSP value. */
10661 if (rc == VINF_SUCCESS)
10662 *pTmpRsp = NewRsp;
10663 }
10664
10665 return rc;
10666}
10667
10668
10669/**
10670 * Pops a dword from the stack, using a temporary stack pointer.
10671 *
10672 * @returns Strict VBox status code.
10673 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10674 * @param pu32Value Where to store the popped value.
10675 * @param pTmpRsp Pointer to the temporary stack pointer.
10676 */
10677IEM_STATIC VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
10678{
10679 /* Increment the stack pointer. */
10680 RTUINT64U NewRsp = *pTmpRsp;
10681 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
10682
10683 /* Write the word the lazy way. */
10684 uint32_t const *pu32Src;
10685 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10686 if (rc == VINF_SUCCESS)
10687 {
10688 *pu32Value = *pu32Src;
10689 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
10690
10691 /* Commit the new RSP value. */
10692 if (rc == VINF_SUCCESS)
10693 *pTmpRsp = NewRsp;
10694 }
10695
10696 return rc;
10697}
10698
10699
10700/**
10701 * Pops a qword from the stack, using a temporary stack pointer.
10702 *
10703 * @returns Strict VBox status code.
10704 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10705 * @param pu64Value Where to store the popped value.
10706 * @param pTmpRsp Pointer to the temporary stack pointer.
10707 */
10708IEM_STATIC VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
10709{
10710 /* Increment the stack pointer. */
10711 RTUINT64U NewRsp = *pTmpRsp;
10712 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
10713
10714 /* Write the word the lazy way. */
10715 uint64_t const *pu64Src;
10716 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10717 if (rcStrict == VINF_SUCCESS)
10718 {
10719 *pu64Value = *pu64Src;
10720 rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
10721
10722 /* Commit the new RSP value. */
10723 if (rcStrict == VINF_SUCCESS)
10724 *pTmpRsp = NewRsp;
10725 }
10726
10727 return rcStrict;
10728}
10729
10730
10731/**
10732 * Begin a special stack push (used by interrupt, exceptions and such).
10733 *
10734 * This will raise \#SS or \#PF if appropriate.
10735 *
10736 * @returns Strict VBox status code.
10737 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10738 * @param cbMem The number of bytes to push onto the stack.
10739 * @param ppvMem Where to return the pointer to the stack memory.
10740 * As with the other memory functions this could be
10741 * direct access or bounce buffered access, so
10742 * don't commit register until the commit call
10743 * succeeds.
10744 * @param puNewRsp Where to return the new RSP value. This must be
10745 * passed unchanged to
10746 * iemMemStackPushCommitSpecial().
10747 */
10748IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp)
10749{
10750 Assert(cbMem < UINT8_MAX);
10751 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
10752 return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10753}
10754
10755
10756/**
10757 * Commits a special stack push (started by iemMemStackPushBeginSpecial).
10758 *
10759 * This will update the rSP.
10760 *
10761 * @returns Strict VBox status code.
10762 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10763 * @param pvMem The pointer returned by
10764 * iemMemStackPushBeginSpecial().
10765 * @param uNewRsp The new RSP value returned by
10766 * iemMemStackPushBeginSpecial().
10767 */
10768IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp)
10769{
10770 VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
10771 if (rcStrict == VINF_SUCCESS)
10772 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10773 return rcStrict;
10774}
10775
10776
10777/**
10778 * Begin a special stack pop (used by iret, retf and such).
10779 *
10780 * This will raise \#SS or \#PF if appropriate.
10781 *
10782 * @returns Strict VBox status code.
10783 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10784 * @param cbMem The number of bytes to pop from the stack.
10785 * @param ppvMem Where to return the pointer to the stack memory.
10786 * @param puNewRsp Where to return the new RSP value. This must be
10787 * assigned to CPUMCTX::rsp manually some time
10788 * after iemMemStackPopDoneSpecial() has been
10789 * called.
10790 */
10791IEM_STATIC VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
10792{
10793 Assert(cbMem < UINT8_MAX);
10794 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
10795 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10796}
10797
10798
10799/**
10800 * Continue a special stack pop (used by iret and retf).
10801 *
10802 * This will raise \#SS or \#PF if appropriate.
10803 *
10804 * @returns Strict VBox status code.
10805 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10806 * @param cbMem The number of bytes to pop from the stack.
10807 * @param ppvMem Where to return the pointer to the stack memory.
10808 * @param puNewRsp Where to return the new RSP value. This must be
10809 * assigned to CPUMCTX::rsp manually some time
10810 * after iemMemStackPopDoneSpecial() has been
10811 * called.
10812 */
10813IEM_STATIC VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
10814{
10815 Assert(cbMem < UINT8_MAX);
10816 RTUINT64U NewRsp;
10817 NewRsp.u = *puNewRsp;
10818 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
10819 *puNewRsp = NewRsp.u;
10820 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10821}
10822
10823
10824/**
10825 * Done with a special stack pop (started by iemMemStackPopBeginSpecial or
10826 * iemMemStackPopContinueSpecial).
10827 *
10828 * The caller will manually commit the rSP.
10829 *
10830 * @returns Strict VBox status code.
10831 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10832 * @param pvMem The pointer returned by
10833 * iemMemStackPopBeginSpecial() or
10834 * iemMemStackPopContinueSpecial().
10835 */
10836IEM_STATIC VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem)
10837{
10838 return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
10839}
10840
10841
10842/**
10843 * Fetches a system table byte.
10844 *
10845 * @returns Strict VBox status code.
10846 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10847 * @param pbDst Where to return the byte.
10848 * @param iSegReg The index of the segment register to use for
10849 * this access. The base and limits are checked.
10850 * @param GCPtrMem The address of the guest memory.
10851 */
10852IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10853{
10854 /* The lazy approach for now... */
10855 uint8_t const *pbSrc;
10856 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10857 if (rc == VINF_SUCCESS)
10858 {
10859 *pbDst = *pbSrc;
10860 rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
10861 }
10862 return rc;
10863}
10864
10865
10866/**
10867 * Fetches a system table word.
10868 *
10869 * @returns Strict VBox status code.
10870 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10871 * @param pu16Dst Where to return the word.
10872 * @param iSegReg The index of the segment register to use for
10873 * this access. The base and limits are checked.
10874 * @param GCPtrMem The address of the guest memory.
10875 */
10876IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10877{
10878 /* The lazy approach for now... */
10879 uint16_t const *pu16Src;
10880 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10881 if (rc == VINF_SUCCESS)
10882 {
10883 *pu16Dst = *pu16Src;
10884 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
10885 }
10886 return rc;
10887}
10888
10889
10890/**
10891 * Fetches a system table dword.
10892 *
10893 * @returns Strict VBox status code.
10894 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10895 * @param pu32Dst Where to return the dword.
10896 * @param iSegReg The index of the segment register to use for
10897 * this access. The base and limits are checked.
10898 * @param GCPtrMem The address of the guest memory.
10899 */
10900IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10901{
10902 /* The lazy approach for now... */
10903 uint32_t const *pu32Src;
10904 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10905 if (rc == VINF_SUCCESS)
10906 {
10907 *pu32Dst = *pu32Src;
10908 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
10909 }
10910 return rc;
10911}
10912
10913
10914/**
10915 * Fetches a system table qword.
10916 *
10917 * @returns Strict VBox status code.
10918 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10919 * @param pu64Dst Where to return the qword.
10920 * @param iSegReg The index of the segment register to use for
10921 * this access. The base and limits are checked.
10922 * @param GCPtrMem The address of the guest memory.
10923 */
10924IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10925{
10926 /* The lazy approach for now... */
10927 uint64_t const *pu64Src;
10928 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10929 if (rc == VINF_SUCCESS)
10930 {
10931 *pu64Dst = *pu64Src;
10932 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
10933 }
10934 return rc;
10935}
10936
10937
10938/**
10939 * Fetches a descriptor table entry with caller specified error code.
10940 *
10941 * @returns Strict VBox status code.
10942 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10943 * @param pDesc Where to return the descriptor table entry.
10944 * @param uSel The selector which table entry to fetch.
10945 * @param uXcpt The exception to raise on table lookup error.
10946 * @param uErrorCode The error code associated with the exception.
10947 */
10948IEM_STATIC VBOXSTRICTRC
10949iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode)
10950{
10951 AssertPtr(pDesc);
10952 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
10953
10954 /** @todo did the 286 require all 8 bytes to be accessible? */
10955 /*
10956 * Get the selector table base and check bounds.
10957 */
10958 RTGCPTR GCPtrBase;
10959 if (uSel & X86_SEL_LDT)
10960 {
10961 if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
10962 || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
10963 {
10964 Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
10965 uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
10966 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
10967 uErrorCode, 0);
10968 }
10969
10970 Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
10971 GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
10972 }
10973 else
10974 {
10975 if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
10976 {
10977 Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
10978 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
10979 uErrorCode, 0);
10980 }
10981 GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
10982 }
10983
10984 /*
10985 * Read the legacy descriptor and maybe the long mode extensions if
10986 * required.
10987 */
10988 VBOXSTRICTRC rcStrict;
10989 if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
10990 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
10991 else
10992 {
10993 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
10994 if (rcStrict == VINF_SUCCESS)
10995 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
10996 if (rcStrict == VINF_SUCCESS)
10997 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
10998 if (rcStrict == VINF_SUCCESS)
10999 pDesc->Legacy.au16[3] = 0;
11000 else
11001 return rcStrict;
11002 }
11003
11004 if (rcStrict == VINF_SUCCESS)
11005 {
11006 if ( !IEM_IS_LONG_MODE(pVCpu)
11007 || pDesc->Legacy.Gen.u1DescType)
11008 pDesc->Long.au64[1] = 0;
11009 else if ((uint32_t)(uSel | X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
11010 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1);
11011 else
11012 {
11013 Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
11014 /** @todo is this the right exception? */
11015 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
11016 }
11017 }
11018 return rcStrict;
11019}
11020
11021
11022/**
11023 * Fetches a descriptor table entry.
11024 *
11025 * @returns Strict VBox status code.
11026 * @param pVCpu The cross context virtual CPU structure of the calling thread.
11027 * @param pDesc Where to return the descriptor table entry.
11028 * @param uSel The selector which table entry to fetch.
11029 * @param uXcpt The exception to raise on table lookup error.
11030 */
11031IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt)
11032{
11033 return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
11034}
11035
11036
11037/**
11038 * Fakes a long mode stack selector for SS = 0.
11039 *
11040 * @param pDescSs Where to return the fake stack descriptor.
11041 * @param uDpl The DPL we want.
11042 */
11043IEM_STATIC void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
11044{
11045 pDescSs->Long.au64[0] = 0;
11046 pDescSs->Long.au64[1] = 0;
11047 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
11048 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
11049 pDescSs->Long.Gen.u2Dpl = uDpl;
11050 pDescSs->Long.Gen.u1Present = 1;
11051 pDescSs->Long.Gen.u1Long = 1;
11052}
11053
11054
11055/**
11056 * Marks the selector descriptor as accessed (only non-system descriptors).
11057 *
11058 * This function ASSUMES that iemMemFetchSelDesc has be called previously and
11059 * will therefore skip the limit checks.
11060 *
11061 * @returns Strict VBox status code.
11062 * @param pVCpu The cross context virtual CPU structure of the calling thread.
11063 * @param uSel The selector.
11064 */
11065IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel)
11066{
11067 /*
11068 * Get the selector table base and calculate the entry address.
11069 */
11070 RTGCPTR GCPtr = uSel & X86_SEL_LDT
11071 ? pVCpu->cpum.GstCtx.ldtr.u64Base
11072 : pVCpu->cpum.GstCtx.gdtr.pGdt;
11073 GCPtr += uSel & X86_SEL_MASK;
11074
11075 /*
11076 * ASMAtomicBitSet will assert if the address is misaligned, so do some
11077 * ugly stuff to avoid this. This will make sure it's an atomic access
11078 * as well more or less remove any question about 8-bit or 32-bit accesss.
11079 */
11080 VBOXSTRICTRC rcStrict;
11081 uint32_t volatile *pu32;
11082 if ((GCPtr & 3) == 0)
11083 {
11084 /* The normal case, map the 32-bit bits around the accessed bit (40). */
11085 GCPtr += 2 + 2;
11086 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
11087 if (rcStrict != VINF_SUCCESS)
11088 return rcStrict;
11089 ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
11090 }
11091 else
11092 {
11093 /* The misaligned GDT/LDT case, map the whole thing. */
11094 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
11095 if (rcStrict != VINF_SUCCESS)
11096 return rcStrict;
11097 switch ((uintptr_t)pu32 & 3)
11098 {
11099 case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
11100 case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
11101 case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
11102 case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
11103 }
11104 }
11105
11106 return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
11107}
11108
11109/** @} */
11110
11111
11112/*
11113 * Include the C/C++ implementation of instruction.
11114 */
11115#include "IEMAllCImpl.cpp.h"
11116
11117
11118
11119/** @name "Microcode" macros.
11120 *
11121 * The idea is that we should be able to use the same code to interpret
11122 * instructions as well as recompiler instructions. Thus this obfuscation.
11123 *
11124 * @{
11125 */
11126#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
11127#define IEM_MC_END() }
11128#define IEM_MC_PAUSE() do {} while (0)
11129#define IEM_MC_CONTINUE() do {} while (0)
11130
11131/** Internal macro. */
11132#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
11133 do \
11134 { \
11135 VBOXSTRICTRC rcStrict2 = a_Expr; \
11136 if (rcStrict2 != VINF_SUCCESS) \
11137 return rcStrict2; \
11138 } while (0)
11139
11140
11141#define IEM_MC_ADVANCE_RIP() iemRegUpdateRipAndClearRF(pVCpu)
11142#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pVCpu, a_i8))
11143#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pVCpu, a_i16))
11144#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pVCpu, a_i32))
11145#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u16NewIP)))
11146#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u32NewIP)))
11147#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u64NewIP)))
11148#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
11149#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
11150 do { \
11151 if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
11152 return iemRaiseDeviceNotAvailable(pVCpu); \
11153 } while (0)
11154#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
11155 do { \
11156 if ((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS)) \
11157 return iemRaiseDeviceNotAvailable(pVCpu); \
11158 } while (0)
11159#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
11160 do { \
11161 if (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES) \
11162 return iemRaiseMathFault(pVCpu); \
11163 } while (0)
11164#define IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT() \
11165 do { \
11166 if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
11167 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
11168 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx2) \
11169 return iemRaiseUndefinedOpcode(pVCpu); \
11170 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11171 return iemRaiseDeviceNotAvailable(pVCpu); \
11172 } while (0)
11173#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
11174 do { \
11175 if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
11176 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
11177 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) \
11178 return iemRaiseUndefinedOpcode(pVCpu); \
11179 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11180 return iemRaiseDeviceNotAvailable(pVCpu); \
11181 } while (0)
11182#define IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT() \
11183 do { \
11184 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11185 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11186 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse41) \
11187 return iemRaiseUndefinedOpcode(pVCpu); \
11188 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11189 return iemRaiseDeviceNotAvailable(pVCpu); \
11190 } while (0)
11191#define IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT() \
11192 do { \
11193 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11194 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11195 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse3) \
11196 return iemRaiseUndefinedOpcode(pVCpu); \
11197 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11198 return iemRaiseDeviceNotAvailable(pVCpu); \
11199 } while (0)
11200#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT() \
11201 do { \
11202 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11203 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11204 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) \
11205 return iemRaiseUndefinedOpcode(pVCpu); \
11206 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11207 return iemRaiseDeviceNotAvailable(pVCpu); \
11208 } while (0)
11209#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
11210 do { \
11211 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11212 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11213 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) \
11214 return iemRaiseUndefinedOpcode(pVCpu); \
11215 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11216 return iemRaiseDeviceNotAvailable(pVCpu); \
11217 } while (0)
11218#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
11219 do { \
11220 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11221 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMmx) \
11222 return iemRaiseUndefinedOpcode(pVCpu); \
11223 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11224 return iemRaiseDeviceNotAvailable(pVCpu); \
11225 } while (0)
11226#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() \
11227 do { \
11228 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11229 || ( !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse \
11230 && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAmdMmxExts) ) \
11231 return iemRaiseUndefinedOpcode(pVCpu); \
11232 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11233 return iemRaiseDeviceNotAvailable(pVCpu); \
11234 } while (0)
11235#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
11236 do { \
11237 if (pVCpu->iem.s.uCpl != 0) \
11238 return iemRaiseGeneralProtectionFault0(pVCpu); \
11239 } while (0)
11240#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
11241 do { \
11242 if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
11243 else return iemRaiseGeneralProtectionFault0(pVCpu); \
11244 } while (0)
11245#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
11246 do { \
11247 if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT \
11248 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFsGsBase \
11249 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE)) \
11250 return iemRaiseUndefinedOpcode(pVCpu); \
11251 } while (0)
11252#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
11253 do { \
11254 if (!IEM_IS_CANONICAL(a_u64Addr)) \
11255 return iemRaiseGeneralProtectionFault0(pVCpu); \
11256 } while (0)
11257
11258
11259#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
11260#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
11261#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
11262#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
11263#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
11264#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
11265#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
11266 uint32_t a_Name; \
11267 uint32_t *a_pName = &a_Name
11268#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
11269 do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
11270
11271#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
11272#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
11273
11274#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11275#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11276#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11277#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11278#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11279#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11280#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11281#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11282#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11283#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11284#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
11285#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
11286#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
11287#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
11288#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
11289#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
11290#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
11291#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
11292 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11293 (a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11294 } while (0)
11295#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
11296 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11297 (a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11298 } while (0)
11299#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
11300 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11301 (a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11302 } while (0)
11303/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
11304#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
11305 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11306 (a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
11307 } while (0)
11308#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
11309 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11310 (a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
11311 } while (0)
11312/** @note Not for IOPL or IF testing or modification. */
11313#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
11314#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
11315#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
11316#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
11317
11318#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
11319#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
11320#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
11321#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
11322#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
11323#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
11324#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
11325#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
11326#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
11327#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
11328/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
11329#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
11330 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11331 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
11332 } while (0)
11333#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
11334 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11335 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); /* clear high bits. */ \
11336 } while (0)
11337#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
11338 do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
11339
11340
11341#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
11342#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
11343/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
11344 * Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
11345#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
11346#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
11347/** @note Not for IOPL or IF testing or modification. */
11348#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.u
11349
11350#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
11351#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
11352#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
11353 do { \
11354 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11355 *pu32Reg += (a_u32Value); \
11356 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11357 } while (0)
11358#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
11359
11360#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
11361#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
11362#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
11363 do { \
11364 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11365 *pu32Reg -= (a_u32Value); \
11366 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11367 } while (0)
11368#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
11369#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
11370
11371#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
11372#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
11373#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
11374#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
11375#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
11376#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
11377#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
11378
11379#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
11380#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
11381#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
11382#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
11383
11384#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
11385#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
11386#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
11387
11388#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) |= (a_u8Mask); } while (0)
11389#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) |= (a_u16Mask); } while (0)
11390#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
11391
11392#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
11393#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
11394#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
11395
11396#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
11397#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
11398#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
11399
11400#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
11401
11402#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
11403
11404#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
11405#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
11406#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
11407 do { \
11408 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11409 *pu32Reg &= (a_u32Value); \
11410 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11411 } while (0)
11412#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
11413
11414#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) |= (a_u8Value)
11415#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) |= (a_u16Value)
11416#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
11417 do { \
11418 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11419 *pu32Reg |= (a_u32Value); \
11420 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11421 } while (0)
11422#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) |= (a_u64Value)
11423
11424
11425/** @note Not for IOPL or IF modification. */
11426#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u |= (a_fBit); } while (0)
11427/** @note Not for IOPL or IF modification. */
11428#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
11429/** @note Not for IOPL or IF modification. */
11430#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
11431
11432#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK | X86_FSW_TOP_MASK; } while (0)
11433
11434/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
11435#define IEM_MC_FPU_TO_MMX_MODE() do { \
11436 pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
11437 pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
11438 } while (0)
11439
11440/** Switches the FPU state from MMX mode (FTW=0xffff). */
11441#define IEM_MC_FPU_FROM_MMX_MODE() do { \
11442 pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
11443 } while (0)
11444
11445#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
11446 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
11447#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
11448 do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
11449#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
11450 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
11451 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
11452 } while (0)
11453#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
11454 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
11455 pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
11456 } while (0)
11457#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
11458 (a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
11459#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
11460 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
11461#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
11462 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
11463
11464#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
11465 do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
11466 (a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
11467 } while (0)
11468#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg) \
11469 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; } while (0)
11470#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg) \
11471 do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0]; } while (0)
11472#define IEM_MC_FETCH_XREG_HI_U64(a_u64Value, a_iXReg) \
11473 do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; } while (0)
11474#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
11475 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
11476 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
11477 } while (0)
11478#define IEM_MC_STORE_XREG_U64(a_iXReg, a_u64Value) \
11479 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); } while (0)
11480#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
11481 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
11482 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
11483 } while (0)
11484#define IEM_MC_STORE_XREG_U32(a_iXReg, a_u32Value) \
11485 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = (a_u32Value); } while (0)
11486#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
11487 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
11488 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
11489 } while (0)
11490#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
11491 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
11492#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
11493 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
11494#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
11495 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
11496#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
11497 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
11498#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
11499 do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
11500 = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
11501 pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
11502 = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
11503 } while (0)
11504
11505#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
11506 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11507 (a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
11508 } while (0)
11509#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
11510 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11511 (a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
11512 } while (0)
11513#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
11514 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11515 (a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
11516 (a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
11517 } while (0)
11518#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
11519 do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11520 (a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
11521 (a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
11522 (a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
11523 (a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
11524 } while (0)
11525
11526#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
11527#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
11528 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11529 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
11530 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
11531 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11532 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11533 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11534 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11535 } while (0)
11536#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
11537 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11538 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
11539 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11540 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11541 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11542 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11543 } while (0)
11544#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
11545 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11546 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
11547 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
11548 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11549 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11550 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11551 } while (0)
11552#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
11553 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11554 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
11555 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
11556 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
11557 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
11558 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11559 } while (0)
11560
11561#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
11562 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
11563#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
11564 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
11565#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
11566 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
11567#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
11568 do { uintptr_t const iYRegTmp = (a_iYReg); \
11569 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
11570 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
11571 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
11572 } while (0)
11573
11574#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11575 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11576 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11577 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
11578 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
11579 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
11580 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
11581 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11582 } while (0)
11583#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11584 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11585 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11586 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
11587 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
11588 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11589 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11590 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11591 } while (0)
11592#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11593 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11594 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11595 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
11596 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11597 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11598 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11599 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11600 } while (0)
11601
11602#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
11603 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11604 uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
11605 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11606 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
11607 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
11608 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11609 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11610 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11611 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11612 } while (0)
11613#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
11614 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11615 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
11616 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11617 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
11618 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11619 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11620 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11621 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11622 } while (0)
11623#define IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
11624 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11625 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
11626 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11627 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
11628 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11629 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11630 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11631 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11632 } while (0)
11633#define IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
11634 do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11635 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11636 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
11637 pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11638 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11639 pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11640 IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
11641 } while (0)
11642
11643#ifndef IEM_WITH_SETJMP
11644# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
11645 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
11646# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
11647 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
11648# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
11649 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
11650#else
11651# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
11652 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11653# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
11654 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
11655# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
11656 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
11657#endif
11658
11659#ifndef IEM_WITH_SETJMP
11660# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11661 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
11662# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11663 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11664# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
11665 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
11666#else
11667# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11668 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11669# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11670 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11671# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
11672 ((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11673#endif
11674
11675#ifndef IEM_WITH_SETJMP
11676# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11677 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
11678# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11679 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11680# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
11681 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
11682#else
11683# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11684 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11685# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11686 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11687# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
11688 ((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11689#endif
11690
11691#ifdef SOME_UNUSED_FUNCTION
11692# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11693 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11694#endif
11695
11696#ifndef IEM_WITH_SETJMP
11697# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11698 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11699# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11700 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11701# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
11702 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11703# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
11704 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
11705#else
11706# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11707 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11708# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11709 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11710# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
11711 ((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11712# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
11713 ((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11714#endif
11715
11716#ifndef IEM_WITH_SETJMP
11717# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
11718 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
11719# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
11720 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
11721# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
11722 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
11723#else
11724# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
11725 ((a_r32Dst).u32 = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11726# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
11727 ((a_r64Dst).au64[0] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11728# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
11729 iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
11730#endif
11731
11732#ifndef IEM_WITH_SETJMP
11733# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
11734 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
11735# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
11736 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
11737#else
11738# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
11739 iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
11740# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
11741 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
11742#endif
11743
11744#ifndef IEM_WITH_SETJMP
11745# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
11746 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
11747# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
11748 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
11749#else
11750# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
11751 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
11752# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
11753 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
11754#endif
11755
11756
11757
11758#ifndef IEM_WITH_SETJMP
11759# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11760 do { \
11761 uint8_t u8Tmp; \
11762 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11763 (a_u16Dst) = u8Tmp; \
11764 } while (0)
11765# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11766 do { \
11767 uint8_t u8Tmp; \
11768 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11769 (a_u32Dst) = u8Tmp; \
11770 } while (0)
11771# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11772 do { \
11773 uint8_t u8Tmp; \
11774 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11775 (a_u64Dst) = u8Tmp; \
11776 } while (0)
11777# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11778 do { \
11779 uint16_t u16Tmp; \
11780 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11781 (a_u32Dst) = u16Tmp; \
11782 } while (0)
11783# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11784 do { \
11785 uint16_t u16Tmp; \
11786 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11787 (a_u64Dst) = u16Tmp; \
11788 } while (0)
11789# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11790 do { \
11791 uint32_t u32Tmp; \
11792 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
11793 (a_u64Dst) = u32Tmp; \
11794 } while (0)
11795#else /* IEM_WITH_SETJMP */
11796# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11797 ((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11798# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11799 ((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11800# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11801 ((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11802# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11803 ((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11804# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11805 ((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11806# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11807 ((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11808#endif /* IEM_WITH_SETJMP */
11809
11810#ifndef IEM_WITH_SETJMP
11811# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11812 do { \
11813 uint8_t u8Tmp; \
11814 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11815 (a_u16Dst) = (int8_t)u8Tmp; \
11816 } while (0)
11817# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11818 do { \
11819 uint8_t u8Tmp; \
11820 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11821 (a_u32Dst) = (int8_t)u8Tmp; \
11822 } while (0)
11823# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11824 do { \
11825 uint8_t u8Tmp; \
11826 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11827 (a_u64Dst) = (int8_t)u8Tmp; \
11828 } while (0)
11829# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11830 do { \
11831 uint16_t u16Tmp; \
11832 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11833 (a_u32Dst) = (int16_t)u16Tmp; \
11834 } while (0)
11835# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11836 do { \
11837 uint16_t u16Tmp; \
11838 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11839 (a_u64Dst) = (int16_t)u16Tmp; \
11840 } while (0)
11841# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11842 do { \
11843 uint32_t u32Tmp; \
11844 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
11845 (a_u64Dst) = (int32_t)u32Tmp; \
11846 } while (0)
11847#else /* IEM_WITH_SETJMP */
11848# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11849 ((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11850# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11851 ((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11852# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11853 ((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11854# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11855 ((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11856# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11857 ((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11858# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11859 ((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11860#endif /* IEM_WITH_SETJMP */
11861
11862#ifndef IEM_WITH_SETJMP
11863# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
11864 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
11865# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
11866 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
11867# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
11868 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
11869# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
11870 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
11871#else
11872# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
11873 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
11874# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
11875 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
11876# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
11877 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
11878# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
11879 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
11880#endif
11881
11882#ifndef IEM_WITH_SETJMP
11883# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
11884 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
11885# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
11886 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
11887# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
11888 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
11889# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
11890 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
11891#else
11892# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
11893 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
11894# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
11895 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
11896# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
11897 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
11898# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
11899 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
11900#endif
11901
11902#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
11903#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
11904#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
11905#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
11906#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u32 = UINT32_C(0xffc00000)
11907#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
11908#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
11909 do { \
11910 (a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
11911 (a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
11912 } while (0)
11913
11914#ifndef IEM_WITH_SETJMP
11915# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
11916 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
11917# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
11918 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
11919#else
11920# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
11921 iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
11922# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
11923 iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
11924#endif
11925
11926#ifndef IEM_WITH_SETJMP
11927# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
11928 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
11929# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
11930 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
11931#else
11932# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
11933 iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
11934# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
11935 iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
11936#endif
11937
11938
11939#define IEM_MC_PUSH_U16(a_u16Value) \
11940 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
11941#define IEM_MC_PUSH_U32(a_u32Value) \
11942 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
11943#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
11944 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_u32Value)))
11945#define IEM_MC_PUSH_U64(a_u64Value) \
11946 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
11947
11948#define IEM_MC_POP_U16(a_pu16Value) \
11949 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
11950#define IEM_MC_POP_U32(a_pu32Value) \
11951 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
11952#define IEM_MC_POP_U64(a_pu64Value) \
11953 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
11954
11955/** Maps guest memory for direct or bounce buffered access.
11956 * The purpose is to pass it to an operand implementation, thus the a_iArg.
11957 * @remarks May return.
11958 */
11959#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
11960 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pMem), sizeof(*(a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
11961
11962/** Maps guest memory for direct or bounce buffered access.
11963 * The purpose is to pass it to an operand implementation, thus the a_iArg.
11964 * @remarks May return.
11965 */
11966#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
11967 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
11968
11969/** Commits the memory and unmaps the guest memory.
11970 * @remarks May return.
11971 */
11972#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
11973 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
11974
11975/** Commits the memory and unmaps the guest memory unless the FPU status word
11976 * indicates (@a a_u16FSW) and FPU control word indicates a pending exception
11977 * that would cause FLD not to store.
11978 *
11979 * The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
11980 * store, while \#P will not.
11981 *
11982 * @remarks May in theory return - for now.
11983 */
11984#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
11985 do { \
11986 if ( !(a_u16FSW & X86_FSW_ES) \
11987 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
11988 & ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
11989 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
11990 } while (0)
11991
11992/** Calculate efficient address from R/M. */
11993#ifndef IEM_WITH_SETJMP
11994# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
11995 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (bRm), (cbImm), &(a_GCPtrEff)))
11996#else
11997# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
11998 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (bRm), (cbImm)))
11999#endif
12000
12001#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
12002#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
12003#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
12004#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
12005#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
12006#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
12007#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
12008
12009/**
12010 * Defers the rest of the instruction emulation to a C implementation routine
12011 * and returns, only taking the standard parameters.
12012 *
12013 * @param a_pfnCImpl The pointer to the C routine.
12014 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
12015 */
12016#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
12017
12018/**
12019 * Defers the rest of instruction emulation to a C implementation routine and
12020 * returns, taking one argument in addition to the standard ones.
12021 *
12022 * @param a_pfnCImpl The pointer to the C routine.
12023 * @param a0 The argument.
12024 */
12025#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
12026
12027/**
12028 * Defers the rest of the instruction emulation to a C implementation routine
12029 * and returns, taking two arguments in addition to the standard ones.
12030 *
12031 * @param a_pfnCImpl The pointer to the C routine.
12032 * @param a0 The first extra argument.
12033 * @param a1 The second extra argument.
12034 */
12035#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
12036
12037/**
12038 * Defers the rest of the instruction emulation to a C implementation routine
12039 * and returns, taking three arguments in addition to the standard ones.
12040 *
12041 * @param a_pfnCImpl The pointer to the C routine.
12042 * @param a0 The first extra argument.
12043 * @param a1 The second extra argument.
12044 * @param a2 The third extra argument.
12045 */
12046#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
12047
12048/**
12049 * Defers the rest of the instruction emulation to a C implementation routine
12050 * and returns, taking four arguments in addition to the standard ones.
12051 *
12052 * @param a_pfnCImpl The pointer to the C routine.
12053 * @param a0 The first extra argument.
12054 * @param a1 The second extra argument.
12055 * @param a2 The third extra argument.
12056 * @param a3 The fourth extra argument.
12057 */
12058#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3)
12059
12060/**
12061 * Defers the rest of the instruction emulation to a C implementation routine
12062 * and returns, taking two arguments in addition to the standard ones.
12063 *
12064 * @param a_pfnCImpl The pointer to the C routine.
12065 * @param a0 The first extra argument.
12066 * @param a1 The second extra argument.
12067 * @param a2 The third extra argument.
12068 * @param a3 The fourth extra argument.
12069 * @param a4 The fifth extra argument.
12070 */
12071#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4)
12072
12073/**
12074 * Defers the entire instruction emulation to a C implementation routine and
12075 * returns, only taking the standard parameters.
12076 *
12077 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12078 *
12079 * @param a_pfnCImpl The pointer to the C routine.
12080 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
12081 */
12082#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
12083
12084/**
12085 * Defers the entire instruction emulation to a C implementation routine and
12086 * returns, taking one argument in addition to the standard ones.
12087 *
12088 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12089 *
12090 * @param a_pfnCImpl The pointer to the C routine.
12091 * @param a0 The argument.
12092 */
12093#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
12094
12095/**
12096 * Defers the entire instruction emulation to a C implementation routine and
12097 * returns, taking two arguments in addition to the standard ones.
12098 *
12099 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12100 *
12101 * @param a_pfnCImpl The pointer to the C routine.
12102 * @param a0 The first extra argument.
12103 * @param a1 The second extra argument.
12104 */
12105#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
12106
12107/**
12108 * Defers the entire instruction emulation to a C implementation routine and
12109 * returns, taking three arguments in addition to the standard ones.
12110 *
12111 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12112 *
12113 * @param a_pfnCImpl The pointer to the C routine.
12114 * @param a0 The first extra argument.
12115 * @param a1 The second extra argument.
12116 * @param a2 The third extra argument.
12117 */
12118#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
12119
12120/**
12121 * Calls a FPU assembly implementation taking one visible argument.
12122 *
12123 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12124 * @param a0 The first extra argument.
12125 */
12126#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
12127 do { \
12128 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
12129 } while (0)
12130
12131/**
12132 * Calls a FPU assembly implementation taking two visible arguments.
12133 *
12134 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12135 * @param a0 The first extra argument.
12136 * @param a1 The second extra argument.
12137 */
12138#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
12139 do { \
12140 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
12141 } while (0)
12142
12143/**
12144 * Calls a FPU assembly implementation taking three visible arguments.
12145 *
12146 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12147 * @param a0 The first extra argument.
12148 * @param a1 The second extra argument.
12149 * @param a2 The third extra argument.
12150 */
12151#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12152 do { \
12153 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
12154 } while (0)
12155
12156#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
12157 do { \
12158 (a_FpuData).FSW = (a_FSW); \
12159 (a_FpuData).r80Result = *(a_pr80Value); \
12160 } while (0)
12161
12162/** Pushes FPU result onto the stack. */
12163#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
12164 iemFpuPushResult(pVCpu, &a_FpuData)
12165/** Pushes FPU result onto the stack and sets the FPUDP. */
12166#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
12167 iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
12168
12169/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
12170#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
12171 iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo)
12172
12173/** Stores FPU result in a stack register. */
12174#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
12175 iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg)
12176/** Stores FPU result in a stack register and pops the stack. */
12177#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
12178 iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg)
12179/** Stores FPU result in a stack register and sets the FPUDP. */
12180#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
12181 iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
12182/** Stores FPU result in a stack register, sets the FPUDP, and pops the
12183 * stack. */
12184#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
12185 iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
12186
12187/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
12188#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
12189 iemFpuUpdateOpcodeAndIp(pVCpu)
12190/** Free a stack register (for FFREE and FFREEP). */
12191#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
12192 iemFpuStackFree(pVCpu, a_iStReg)
12193/** Increment the FPU stack pointer. */
12194#define IEM_MC_FPU_STACK_INC_TOP() \
12195 iemFpuStackIncTop(pVCpu)
12196/** Decrement the FPU stack pointer. */
12197#define IEM_MC_FPU_STACK_DEC_TOP() \
12198 iemFpuStackDecTop(pVCpu)
12199
12200/** Updates the FSW, FOP, FPUIP, and FPUCS. */
12201#define IEM_MC_UPDATE_FSW(a_u16FSW) \
12202 iemFpuUpdateFSW(pVCpu, a_u16FSW)
12203/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
12204#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
12205 iemFpuUpdateFSW(pVCpu, a_u16FSW)
12206/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
12207#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
12208 iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
12209/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
12210#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
12211 iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW)
12212/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
12213 * stack. */
12214#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
12215 iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
12216/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
12217#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
12218 iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW)
12219
12220/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
12221#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
12222 iemFpuStackUnderflow(pVCpu, a_iStDst)
12223/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
12224 * stack. */
12225#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
12226 iemFpuStackUnderflowThenPop(pVCpu, a_iStDst)
12227/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
12228 * FPUDS. */
12229#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
12230 iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
12231/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
12232 * FPUDS. Pops stack. */
12233#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
12234 iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
12235/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
12236 * stack twice. */
12237#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
12238 iemFpuStackUnderflowThenPopPop(pVCpu)
12239/** Raises a FPU stack underflow exception for an instruction pushing a result
12240 * value onto the stack. Sets FPUIP, FPUCS and FOP. */
12241#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
12242 iemFpuStackPushUnderflow(pVCpu)
12243/** Raises a FPU stack underflow exception for an instruction pushing a result
12244 * value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
12245#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
12246 iemFpuStackPushUnderflowTwo(pVCpu)
12247
12248/** Raises a FPU stack overflow exception as part of a push attempt. Sets
12249 * FPUIP, FPUCS and FOP. */
12250#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
12251 iemFpuStackPushOverflow(pVCpu)
12252/** Raises a FPU stack overflow exception as part of a push attempt. Sets
12253 * FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
12254#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
12255 iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff)
12256/** Prepares for using the FPU state.
12257 * Ensures that we can use the host FPU in the current context (RC+R0.
12258 * Ensures the guest FPU state in the CPUMCTX is up to date. */
12259#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
12260/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
12261#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
12262/** Actualizes the guest FPU state so it can be accessed and modified. */
12263#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
12264
12265/** Prepares for using the SSE state.
12266 * Ensures that we can use the host SSE/FPU in the current context (RC+R0.
12267 * Ensures the guest SSE state in the CPUMCTX is up to date. */
12268#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
12269/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
12270#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
12271/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
12272#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
12273
12274/** Prepares for using the AVX state.
12275 * Ensures that we can use the host AVX/FPU in the current context (RC+R0.
12276 * Ensures the guest AVX state in the CPUMCTX is up to date.
12277 * @note This will include the AVX512 state too when support for it is added
12278 * due to the zero extending feature of VEX instruction. */
12279#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
12280/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
12281#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
12282/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
12283#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
12284
12285/**
12286 * Calls a MMX assembly implementation taking two visible arguments.
12287 *
12288 * @param a_pfnAImpl Pointer to the assembly MMX routine.
12289 * @param a0 The first extra argument.
12290 * @param a1 The second extra argument.
12291 */
12292#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
12293 do { \
12294 IEM_MC_PREPARE_FPU_USAGE(); \
12295 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
12296 } while (0)
12297
12298/**
12299 * Calls a MMX assembly implementation taking three visible arguments.
12300 *
12301 * @param a_pfnAImpl Pointer to the assembly MMX routine.
12302 * @param a0 The first extra argument.
12303 * @param a1 The second extra argument.
12304 * @param a2 The third extra argument.
12305 */
12306#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12307 do { \
12308 IEM_MC_PREPARE_FPU_USAGE(); \
12309 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
12310 } while (0)
12311
12312
12313/**
12314 * Calls a SSE assembly implementation taking two visible arguments.
12315 *
12316 * @param a_pfnAImpl Pointer to the assembly SSE routine.
12317 * @param a0 The first extra argument.
12318 * @param a1 The second extra argument.
12319 */
12320#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
12321 do { \
12322 IEM_MC_PREPARE_SSE_USAGE(); \
12323 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
12324 } while (0)
12325
12326/**
12327 * Calls a SSE assembly implementation taking three visible arguments.
12328 *
12329 * @param a_pfnAImpl Pointer to the assembly SSE routine.
12330 * @param a0 The first extra argument.
12331 * @param a1 The second extra argument.
12332 * @param a2 The third extra argument.
12333 */
12334#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12335 do { \
12336 IEM_MC_PREPARE_SSE_USAGE(); \
12337 a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
12338 } while (0)
12339
12340
12341/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
12342 * IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
12343#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
12344 IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
12345
12346/**
12347 * Calls a AVX assembly implementation taking two visible arguments.
12348 *
12349 * There is one implicit zero'th argument, a pointer to the extended state.
12350 *
12351 * @param a_pfnAImpl Pointer to the assembly AVX routine.
12352 * @param a1 The first extra argument.
12353 * @param a2 The second extra argument.
12354 */
12355#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
12356 do { \
12357 IEM_MC_PREPARE_AVX_USAGE(); \
12358 a_pfnAImpl(pXState, (a1), (a2)); \
12359 } while (0)
12360
12361/**
12362 * Calls a AVX assembly implementation taking three visible arguments.
12363 *
12364 * There is one implicit zero'th argument, a pointer to the extended state.
12365 *
12366 * @param a_pfnAImpl Pointer to the assembly AVX routine.
12367 * @param a1 The first extra argument.
12368 * @param a2 The second extra argument.
12369 * @param a3 The third extra argument.
12370 */
12371#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
12372 do { \
12373 IEM_MC_PREPARE_AVX_USAGE(); \
12374 a_pfnAImpl(pXState, (a1), (a2), (a3)); \
12375 } while (0)
12376
12377/** @note Not for IOPL or IF testing. */
12378#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
12379/** @note Not for IOPL or IF testing. */
12380#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
12381/** @note Not for IOPL or IF testing. */
12382#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
12383/** @note Not for IOPL or IF testing. */
12384#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
12385/** @note Not for IOPL or IF testing. */
12386#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
12387 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12388 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12389/** @note Not for IOPL or IF testing. */
12390#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
12391 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12392 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12393/** @note Not for IOPL or IF testing. */
12394#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
12395 if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
12396 || !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12397 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12398/** @note Not for IOPL or IF testing. */
12399#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
12400 if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
12401 && !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12402 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12403#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
12404#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
12405#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
12406/** @note Not for IOPL or IF testing. */
12407#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12408 if ( pVCpu->cpum.GstCtx.cx != 0 \
12409 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12410/** @note Not for IOPL or IF testing. */
12411#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12412 if ( pVCpu->cpum.GstCtx.ecx != 0 \
12413 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12414/** @note Not for IOPL or IF testing. */
12415#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12416 if ( pVCpu->cpum.GstCtx.rcx != 0 \
12417 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12418/** @note Not for IOPL or IF testing. */
12419#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12420 if ( pVCpu->cpum.GstCtx.cx != 0 \
12421 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12422/** @note Not for IOPL or IF testing. */
12423#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12424 if ( pVCpu->cpum.GstCtx.ecx != 0 \
12425 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12426/** @note Not for IOPL or IF testing. */
12427#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12428 if ( pVCpu->cpum.GstCtx.rcx != 0 \
12429 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12430#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
12431#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
12432
12433#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
12434 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
12435#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
12436 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
12437#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
12438 if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
12439#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
12440 if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
12441#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
12442 if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
12443#define IEM_MC_IF_FCW_IM() \
12444 if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
12445
12446#define IEM_MC_ELSE() } else {
12447#define IEM_MC_ENDIF() } do {} while (0)
12448
12449/** @} */
12450
12451
12452/** @name Opcode Debug Helpers.
12453 * @{
12454 */
12455#ifdef VBOX_WITH_STATISTICS
12456# ifdef IN_RING3
12457# define IEMOP_INC_STATS(a_Stats) do { pVCpu->iem.s.StatsR3.a_Stats += 1; } while (0)
12458# else
12459# define IEMOP_INC_STATS(a_Stats) do { pVCpu->iem.s.StatsRZ.a_Stats += 1; } while (0)
12460# endif
12461#else
12462# define IEMOP_INC_STATS(a_Stats) do { } while (0)
12463#endif
12464
12465#ifdef DEBUG
12466# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) \
12467 do { \
12468 IEMOP_INC_STATS(a_Stats); \
12469 Log4(("decode - %04x:%RGv %s%s [#%u]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, \
12470 pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pVCpu->iem.s.cInstructions)); \
12471 } while (0)
12472
12473# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12474 do { \
12475 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12476 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12477 (void)RT_CONCAT(OP_,a_Upper); \
12478 (void)(a_fDisHints); \
12479 (void)(a_fIemHints); \
12480 } while (0)
12481
12482# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12483 do { \
12484 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12485 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12486 (void)RT_CONCAT(OP_,a_Upper); \
12487 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12488 (void)(a_fDisHints); \
12489 (void)(a_fIemHints); \
12490 } while (0)
12491
12492# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12493 do { \
12494 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12495 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12496 (void)RT_CONCAT(OP_,a_Upper); \
12497 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12498 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12499 (void)(a_fDisHints); \
12500 (void)(a_fIemHints); \
12501 } while (0)
12502
12503# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12504 do { \
12505 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12506 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12507 (void)RT_CONCAT(OP_,a_Upper); \
12508 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12509 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12510 (void)RT_CONCAT(OP_PARM_,a_Op3); \
12511 (void)(a_fDisHints); \
12512 (void)(a_fIemHints); \
12513 } while (0)
12514
12515# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12516 do { \
12517 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12518 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12519 (void)RT_CONCAT(OP_,a_Upper); \
12520 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12521 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12522 (void)RT_CONCAT(OP_PARM_,a_Op3); \
12523 (void)RT_CONCAT(OP_PARM_,a_Op4); \
12524 (void)(a_fDisHints); \
12525 (void)(a_fIemHints); \
12526 } while (0)
12527
12528#else
12529# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats)
12530
12531# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12532 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12533# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12534 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12535# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12536 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12537# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12538 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12539# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12540 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12541
12542#endif
12543
12544#define IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12545 IEMOP_MNEMONIC0EX(a_Lower, \
12546 #a_Lower, \
12547 a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
12548#define IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12549 IEMOP_MNEMONIC1EX(RT_CONCAT3(a_Lower,_,a_Op1), \
12550 #a_Lower " " #a_Op1, \
12551 a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
12552#define IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12553 IEMOP_MNEMONIC2EX(RT_CONCAT5(a_Lower,_,a_Op1,_,a_Op2), \
12554 #a_Lower " " #a_Op1 "," #a_Op2, \
12555 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
12556#define IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12557 IEMOP_MNEMONIC3EX(RT_CONCAT7(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3), \
12558 #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3, \
12559 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
12560#define IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12561 IEMOP_MNEMONIC4EX(RT_CONCAT9(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3,_,a_Op4), \
12562 #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3 "," #a_Op4, \
12563 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints)
12564
12565/** @} */
12566
12567
12568/** @name Opcode Helpers.
12569 * @{
12570 */
12571
12572#ifdef IN_RING3
12573# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
12574 do { \
12575 if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
12576 else \
12577 { \
12578 (void)DBGFSTOP(pVCpu->CTX_SUFF(pVM)); \
12579 return IEMOP_RAISE_INVALID_OPCODE(); \
12580 } \
12581 } while (0)
12582#else
12583# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
12584 do { \
12585 if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
12586 else return IEMOP_RAISE_INVALID_OPCODE(); \
12587 } while (0)
12588#endif
12589
12590/** The instruction requires a 186 or later. */
12591#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_186
12592# define IEMOP_HLP_MIN_186() do { } while (0)
12593#else
12594# define IEMOP_HLP_MIN_186() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_186, true)
12595#endif
12596
12597/** The instruction requires a 286 or later. */
12598#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_286
12599# define IEMOP_HLP_MIN_286() do { } while (0)
12600#else
12601# define IEMOP_HLP_MIN_286() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_286, true)
12602#endif
12603
12604/** The instruction requires a 386 or later. */
12605#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
12606# define IEMOP_HLP_MIN_386() do { } while (0)
12607#else
12608# define IEMOP_HLP_MIN_386() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, true)
12609#endif
12610
12611/** The instruction requires a 386 or later if the given expression is true. */
12612#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
12613# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) do { } while (0)
12614#else
12615# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, a_fOnlyIf)
12616#endif
12617
12618/** The instruction requires a 486 or later. */
12619#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_486
12620# define IEMOP_HLP_MIN_486() do { } while (0)
12621#else
12622# define IEMOP_HLP_MIN_486() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_486, true)
12623#endif
12624
12625/** The instruction requires a Pentium (586) or later. */
12626#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PENTIUM
12627# define IEMOP_HLP_MIN_586() do { } while (0)
12628#else
12629# define IEMOP_HLP_MIN_586() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PENTIUM, true)
12630#endif
12631
12632/** The instruction requires a PentiumPro (686) or later. */
12633#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PPRO
12634# define IEMOP_HLP_MIN_686() do { } while (0)
12635#else
12636# define IEMOP_HLP_MIN_686() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PPRO, true)
12637#endif
12638
12639
12640/** The instruction raises an \#UD in real and V8086 mode. */
12641#define IEMOP_HLP_NO_REAL_OR_V86_MODE() \
12642 do \
12643 { \
12644 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu)) { /* likely */ } \
12645 else return IEMOP_RAISE_INVALID_OPCODE(); \
12646 } while (0)
12647
12648#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12649/** This instruction raises an \#UD in real and V8086 mode or when not using a
12650 * 64-bit code segment when in long mode (applicable to all VMX instructions
12651 * except VMCALL).
12652 */
12653#define IEMOP_HLP_VMX_INSTR(a_szInstr, a_InsDiagPrefix) \
12654 do \
12655 { \
12656 if ( !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
12657 && ( !IEM_IS_LONG_MODE(pVCpu) \
12658 || IEM_IS_64BIT_CODE(pVCpu))) \
12659 { /* likely */ } \
12660 else \
12661 { \
12662 if (IEM_IS_REAL_OR_V86_MODE(pVCpu)) \
12663 { \
12664 pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_RealOrV86Mode; \
12665 Log5((a_szInstr ": Real or v8086 mode -> #UD\n")); \
12666 return IEMOP_RAISE_INVALID_OPCODE(); \
12667 } \
12668 if (IEM_IS_LONG_MODE(pVCpu) && !IEM_IS_64BIT_CODE(pVCpu)) \
12669 { \
12670 pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_LongModeCS; \
12671 Log5((a_szInstr ": Long mode without 64-bit code segment -> #UD\n")); \
12672 return IEMOP_RAISE_INVALID_OPCODE(); \
12673 } \
12674 } \
12675 } while (0)
12676
12677/** The instruction can only be executed in VMX operation (VMX root mode and
12678 * non-root mode).
12679 *
12680 * @note Update IEM_VMX_IN_VMX_OPERATION if changes are made here.
12681 */
12682# define IEMOP_HLP_IN_VMX_OPERATION(a_szInstr, a_InsDiagPrefix) \
12683 do \
12684 { \
12685 if (IEM_VMX_IS_ROOT_MODE(pVCpu)) { /* likely */ } \
12686 else \
12687 { \
12688 pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_VmxRoot; \
12689 Log5((a_szInstr ": Not in VMX operation (root mode) -> #UD\n")); \
12690 return IEMOP_RAISE_INVALID_OPCODE(); \
12691 } \
12692 } while (0)
12693#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12694
12695/** The instruction is not available in 64-bit mode, throw \#UD if we're in
12696 * 64-bit mode. */
12697#define IEMOP_HLP_NO_64BIT() \
12698 do \
12699 { \
12700 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12701 return IEMOP_RAISE_INVALID_OPCODE(); \
12702 } while (0)
12703
12704/** The instruction is only available in 64-bit mode, throw \#UD if we're not in
12705 * 64-bit mode. */
12706#define IEMOP_HLP_ONLY_64BIT() \
12707 do \
12708 { \
12709 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) \
12710 return IEMOP_RAISE_INVALID_OPCODE(); \
12711 } while (0)
12712
12713/** The instruction defaults to 64-bit operand size if 64-bit mode. */
12714#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
12715 do \
12716 { \
12717 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12718 iemRecalEffOpSize64Default(pVCpu); \
12719 } while (0)
12720
12721/** The instruction has 64-bit operand size if 64-bit mode. */
12722#define IEMOP_HLP_64BIT_OP_SIZE() \
12723 do \
12724 { \
12725 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12726 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; \
12727 } while (0)
12728
12729/** Only a REX prefix immediately preceeding the first opcode byte takes
12730 * effect. This macro helps ensuring this as well as logging bad guest code. */
12731#define IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE(a_szPrf) \
12732 do \
12733 { \
12734 if (RT_UNLIKELY(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX)) \
12735 { \
12736 Log5((a_szPrf ": Overriding REX prefix at %RX16! fPrefixes=%#x\n", pVCpu->cpum.GstCtx.rip, pVCpu->iem.s.fPrefixes)); \
12737 pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REX_MASK; \
12738 pVCpu->iem.s.uRexB = 0; \
12739 pVCpu->iem.s.uRexIndex = 0; \
12740 pVCpu->iem.s.uRexReg = 0; \
12741 iemRecalEffOpSize(pVCpu); \
12742 } \
12743 } while (0)
12744
12745/**
12746 * Done decoding.
12747 */
12748#define IEMOP_HLP_DONE_DECODING() \
12749 do \
12750 { \
12751 /*nothing for now, maybe later... */ \
12752 } while (0)
12753
12754/**
12755 * Done decoding, raise \#UD exception if lock prefix present.
12756 */
12757#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
12758 do \
12759 { \
12760 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12761 { /* likely */ } \
12762 else \
12763 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12764 } while (0)
12765
12766
12767/**
12768 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12769 * repnz or size prefixes are present, or if in real or v8086 mode.
12770 */
12771#define IEMOP_HLP_DONE_VEX_DECODING() \
12772 do \
12773 { \
12774 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12775 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12776 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
12777 { /* likely */ } \
12778 else \
12779 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12780 } while (0)
12781
12782/**
12783 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12784 * repnz or size prefixes are present, or if in real or v8086 mode.
12785 */
12786#define IEMOP_HLP_DONE_VEX_DECODING_L0() \
12787 do \
12788 { \
12789 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12790 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12791 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
12792 && pVCpu->iem.s.uVexLength == 0)) \
12793 { /* likely */ } \
12794 else \
12795 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12796 } while (0)
12797
12798
12799/**
12800 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12801 * repnz or size prefixes are present, or if the VEX.VVVV field doesn't indicate
12802 * register 0, or if in real or v8086 mode.
12803 */
12804#define IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV() \
12805 do \
12806 { \
12807 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12808 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12809 && !pVCpu->iem.s.uVex3rdReg \
12810 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
12811 { /* likely */ } \
12812 else \
12813 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12814 } while (0)
12815
12816/**
12817 * Done decoding VEX, no V, L=0.
12818 * Raises \#UD exception if rex, rep, opsize or lock prefixes are present, if
12819 * we're in real or v8086 mode, if VEX.V!=0xf, or if VEX.L!=0.
12820 */
12821#define IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV() \
12822 do \
12823 { \
12824 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12825 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REX)) \
12826 && pVCpu->iem.s.uVexLength == 0 \
12827 && pVCpu->iem.s.uVex3rdReg == 0 \
12828 && !IEM_IS_REAL_OR_V86_MODE(pVCpu))) \
12829 { /* likely */ } \
12830 else \
12831 return IEMOP_RAISE_INVALID_OPCODE(); \
12832 } while (0)
12833
12834#define IEMOP_HLP_DECODED_NL_1(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_fDisOpType) \
12835 do \
12836 { \
12837 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12838 { /* likely */ } \
12839 else \
12840 { \
12841 NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_fDisOpType); \
12842 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12843 } \
12844 } while (0)
12845#define IEMOP_HLP_DECODED_NL_2(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_uDisParam1, a_fDisOpType) \
12846 do \
12847 { \
12848 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12849 { /* likely */ } \
12850 else \
12851 { \
12852 NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_uDisParam1); NOREF(a_fDisOpType); \
12853 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12854 } \
12855 } while (0)
12856
12857/**
12858 * Done decoding, raise \#UD exception if any lock, repz or repnz prefixes
12859 * are present.
12860 */
12861#define IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES() \
12862 do \
12863 { \
12864 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
12865 { /* likely */ } \
12866 else \
12867 return IEMOP_RAISE_INVALID_OPCODE(); \
12868 } while (0)
12869
12870/**
12871 * Done decoding, raise \#UD exception if any operand-size override, repz or repnz
12872 * prefixes are present.
12873 */
12874#define IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES() \
12875 do \
12876 { \
12877 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
12878 { /* likely */ } \
12879 else \
12880 return IEMOP_RAISE_INVALID_OPCODE(); \
12881 } while (0)
12882
12883
12884/**
12885 * Calculates the effective address of a ModR/M memory operand.
12886 *
12887 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
12888 *
12889 * @return Strict VBox status code.
12890 * @param pVCpu The cross context virtual CPU structure of the calling thread.
12891 * @param bRm The ModRM byte.
12892 * @param cbImm The size of any immediate following the
12893 * effective address opcode bytes. Important for
12894 * RIP relative addressing.
12895 * @param pGCPtrEff Where to return the effective address.
12896 */
12897IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff)
12898{
12899 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
12900# define SET_SS_DEF() \
12901 do \
12902 { \
12903 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
12904 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
12905 } while (0)
12906
12907 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
12908 {
12909/** @todo Check the effective address size crap! */
12910 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
12911 {
12912 uint16_t u16EffAddr;
12913
12914 /* Handle the disp16 form with no registers first. */
12915 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
12916 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
12917 else
12918 {
12919 /* Get the displacment. */
12920 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
12921 {
12922 case 0: u16EffAddr = 0; break;
12923 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
12924 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
12925 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
12926 }
12927
12928 /* Add the base and index registers to the disp. */
12929 switch (bRm & X86_MODRM_RM_MASK)
12930 {
12931 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
12932 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
12933 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
12934 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
12935 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
12936 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
12937 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
12938 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
12939 }
12940 }
12941
12942 *pGCPtrEff = u16EffAddr;
12943 }
12944 else
12945 {
12946 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
12947 uint32_t u32EffAddr;
12948
12949 /* Handle the disp32 form with no registers first. */
12950 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
12951 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
12952 else
12953 {
12954 /* Get the register (or SIB) value. */
12955 switch ((bRm & X86_MODRM_RM_MASK))
12956 {
12957 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
12958 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
12959 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
12960 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
12961 case 4: /* SIB */
12962 {
12963 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
12964
12965 /* Get the index and scale it. */
12966 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
12967 {
12968 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
12969 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
12970 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
12971 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
12972 case 4: u32EffAddr = 0; /*none */ break;
12973 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
12974 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
12975 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
12976 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12977 }
12978 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
12979
12980 /* add base */
12981 switch (bSib & X86_SIB_BASE_MASK)
12982 {
12983 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
12984 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
12985 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
12986 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
12987 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
12988 case 5:
12989 if ((bRm & X86_MODRM_MOD_MASK) != 0)
12990 {
12991 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
12992 SET_SS_DEF();
12993 }
12994 else
12995 {
12996 uint32_t u32Disp;
12997 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
12998 u32EffAddr += u32Disp;
12999 }
13000 break;
13001 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13002 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13003 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13004 }
13005 break;
13006 }
13007 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13008 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13009 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13010 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13011 }
13012
13013 /* Get and add the displacement. */
13014 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13015 {
13016 case 0:
13017 break;
13018 case 1:
13019 {
13020 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13021 u32EffAddr += i8Disp;
13022 break;
13023 }
13024 case 2:
13025 {
13026 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13027 u32EffAddr += u32Disp;
13028 break;
13029 }
13030 default:
13031 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
13032 }
13033
13034 }
13035 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13036 *pGCPtrEff = u32EffAddr;
13037 else
13038 {
13039 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13040 *pGCPtrEff = u32EffAddr & UINT16_MAX;
13041 }
13042 }
13043 }
13044 else
13045 {
13046 uint64_t u64EffAddr;
13047
13048 /* Handle the rip+disp32 form with no registers first. */
13049 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13050 {
13051 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13052 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13053 }
13054 else
13055 {
13056 /* Get the register (or SIB) value. */
13057 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13058 {
13059 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13060 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13061 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13062 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13063 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13064 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13065 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13066 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13067 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13068 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13069 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13070 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13071 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13072 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13073 /* SIB */
13074 case 4:
13075 case 12:
13076 {
13077 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13078
13079 /* Get the index and scale it. */
13080 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13081 {
13082 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13083 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13084 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13085 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13086 case 4: u64EffAddr = 0; /*none */ break;
13087 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13088 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13089 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13090 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13091 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13092 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13093 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13094 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13095 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13096 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13097 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13098 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13099 }
13100 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13101
13102 /* add base */
13103 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13104 {
13105 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13106 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13107 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13108 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13109 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
13110 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13111 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13112 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13113 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13114 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13115 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13116 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13117 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13118 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13119 /* complicated encodings */
13120 case 5:
13121 case 13:
13122 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13123 {
13124 if (!pVCpu->iem.s.uRexB)
13125 {
13126 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13127 SET_SS_DEF();
13128 }
13129 else
13130 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13131 }
13132 else
13133 {
13134 uint32_t u32Disp;
13135 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13136 u64EffAddr += (int32_t)u32Disp;
13137 }
13138 break;
13139 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13140 }
13141 break;
13142 }
13143 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13144 }
13145
13146 /* Get and add the displacement. */
13147 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13148 {
13149 case 0:
13150 break;
13151 case 1:
13152 {
13153 int8_t i8Disp;
13154 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13155 u64EffAddr += i8Disp;
13156 break;
13157 }
13158 case 2:
13159 {
13160 uint32_t u32Disp;
13161 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13162 u64EffAddr += (int32_t)u32Disp;
13163 break;
13164 }
13165 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
13166 }
13167
13168 }
13169
13170 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13171 *pGCPtrEff = u64EffAddr;
13172 else
13173 {
13174 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13175 *pGCPtrEff = u64EffAddr & UINT32_MAX;
13176 }
13177 }
13178
13179 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
13180 return VINF_SUCCESS;
13181}
13182
13183
13184/**
13185 * Calculates the effective address of a ModR/M memory operand.
13186 *
13187 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
13188 *
13189 * @return Strict VBox status code.
13190 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13191 * @param bRm The ModRM byte.
13192 * @param cbImm The size of any immediate following the
13193 * effective address opcode bytes. Important for
13194 * RIP relative addressing.
13195 * @param pGCPtrEff Where to return the effective address.
13196 * @param offRsp RSP displacement.
13197 */
13198IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp)
13199{
13200 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
13201# define SET_SS_DEF() \
13202 do \
13203 { \
13204 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
13205 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
13206 } while (0)
13207
13208 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
13209 {
13210/** @todo Check the effective address size crap! */
13211 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
13212 {
13213 uint16_t u16EffAddr;
13214
13215 /* Handle the disp16 form with no registers first. */
13216 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
13217 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
13218 else
13219 {
13220 /* Get the displacment. */
13221 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13222 {
13223 case 0: u16EffAddr = 0; break;
13224 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
13225 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
13226 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
13227 }
13228
13229 /* Add the base and index registers to the disp. */
13230 switch (bRm & X86_MODRM_RM_MASK)
13231 {
13232 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
13233 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
13234 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
13235 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
13236 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
13237 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
13238 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
13239 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
13240 }
13241 }
13242
13243 *pGCPtrEff = u16EffAddr;
13244 }
13245 else
13246 {
13247 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13248 uint32_t u32EffAddr;
13249
13250 /* Handle the disp32 form with no registers first. */
13251 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13252 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
13253 else
13254 {
13255 /* Get the register (or SIB) value. */
13256 switch ((bRm & X86_MODRM_RM_MASK))
13257 {
13258 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13259 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13260 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13261 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13262 case 4: /* SIB */
13263 {
13264 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13265
13266 /* Get the index and scale it. */
13267 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
13268 {
13269 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13270 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13271 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13272 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13273 case 4: u32EffAddr = 0; /*none */ break;
13274 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
13275 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13276 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13277 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13278 }
13279 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13280
13281 /* add base */
13282 switch (bSib & X86_SIB_BASE_MASK)
13283 {
13284 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
13285 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
13286 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
13287 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
13288 case 4:
13289 u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
13290 SET_SS_DEF();
13291 break;
13292 case 5:
13293 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13294 {
13295 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
13296 SET_SS_DEF();
13297 }
13298 else
13299 {
13300 uint32_t u32Disp;
13301 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13302 u32EffAddr += u32Disp;
13303 }
13304 break;
13305 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13306 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13307 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13308 }
13309 break;
13310 }
13311 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13312 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13313 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13314 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13315 }
13316
13317 /* Get and add the displacement. */
13318 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13319 {
13320 case 0:
13321 break;
13322 case 1:
13323 {
13324 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13325 u32EffAddr += i8Disp;
13326 break;
13327 }
13328 case 2:
13329 {
13330 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13331 u32EffAddr += u32Disp;
13332 break;
13333 }
13334 default:
13335 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
13336 }
13337
13338 }
13339 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13340 *pGCPtrEff = u32EffAddr;
13341 else
13342 {
13343 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13344 *pGCPtrEff = u32EffAddr & UINT16_MAX;
13345 }
13346 }
13347 }
13348 else
13349 {
13350 uint64_t u64EffAddr;
13351
13352 /* Handle the rip+disp32 form with no registers first. */
13353 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13354 {
13355 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13356 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13357 }
13358 else
13359 {
13360 /* Get the register (or SIB) value. */
13361 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13362 {
13363 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13364 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13365 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13366 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13367 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13368 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13369 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13370 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13371 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13372 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13373 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13374 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13375 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13376 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13377 /* SIB */
13378 case 4:
13379 case 12:
13380 {
13381 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13382
13383 /* Get the index and scale it. */
13384 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13385 {
13386 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13387 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13388 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13389 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13390 case 4: u64EffAddr = 0; /*none */ break;
13391 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13392 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13393 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13394 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13395 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13396 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13397 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13398 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13399 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13400 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13401 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13402 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13403 }
13404 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13405
13406 /* add base */
13407 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13408 {
13409 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13410 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13411 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13412 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13413 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
13414 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13415 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13416 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13417 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13418 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13419 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13420 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13421 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13422 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13423 /* complicated encodings */
13424 case 5:
13425 case 13:
13426 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13427 {
13428 if (!pVCpu->iem.s.uRexB)
13429 {
13430 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13431 SET_SS_DEF();
13432 }
13433 else
13434 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13435 }
13436 else
13437 {
13438 uint32_t u32Disp;
13439 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13440 u64EffAddr += (int32_t)u32Disp;
13441 }
13442 break;
13443 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13444 }
13445 break;
13446 }
13447 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13448 }
13449
13450 /* Get and add the displacement. */
13451 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13452 {
13453 case 0:
13454 break;
13455 case 1:
13456 {
13457 int8_t i8Disp;
13458 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13459 u64EffAddr += i8Disp;
13460 break;
13461 }
13462 case 2:
13463 {
13464 uint32_t u32Disp;
13465 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13466 u64EffAddr += (int32_t)u32Disp;
13467 break;
13468 }
13469 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
13470 }
13471
13472 }
13473
13474 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13475 *pGCPtrEff = u64EffAddr;
13476 else
13477 {
13478 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13479 *pGCPtrEff = u64EffAddr & UINT32_MAX;
13480 }
13481 }
13482
13483 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
13484 return VINF_SUCCESS;
13485}
13486
13487
13488#ifdef IEM_WITH_SETJMP
13489/**
13490 * Calculates the effective address of a ModR/M memory operand.
13491 *
13492 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
13493 *
13494 * May longjmp on internal error.
13495 *
13496 * @return The effective address.
13497 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13498 * @param bRm The ModRM byte.
13499 * @param cbImm The size of any immediate following the
13500 * effective address opcode bytes. Important for
13501 * RIP relative addressing.
13502 */
13503IEM_STATIC RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm)
13504{
13505 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
13506# define SET_SS_DEF() \
13507 do \
13508 { \
13509 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
13510 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
13511 } while (0)
13512
13513 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
13514 {
13515/** @todo Check the effective address size crap! */
13516 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
13517 {
13518 uint16_t u16EffAddr;
13519
13520 /* Handle the disp16 form with no registers first. */
13521 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
13522 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
13523 else
13524 {
13525 /* Get the displacment. */
13526 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13527 {
13528 case 0: u16EffAddr = 0; break;
13529 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
13530 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
13531 default: AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); /* (caller checked for these) */
13532 }
13533
13534 /* Add the base and index registers to the disp. */
13535 switch (bRm & X86_MODRM_RM_MASK)
13536 {
13537 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
13538 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
13539 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
13540 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
13541 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
13542 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
13543 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
13544 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
13545 }
13546 }
13547
13548 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
13549 return u16EffAddr;
13550 }
13551
13552 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13553 uint32_t u32EffAddr;
13554
13555 /* Handle the disp32 form with no registers first. */
13556 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13557 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
13558 else
13559 {
13560 /* Get the register (or SIB) value. */
13561 switch ((bRm & X86_MODRM_RM_MASK))
13562 {
13563 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13564 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13565 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13566 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13567 case 4: /* SIB */
13568 {
13569 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13570
13571 /* Get the index and scale it. */
13572 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
13573 {
13574 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13575 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13576 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13577 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13578 case 4: u32EffAddr = 0; /*none */ break;
13579 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
13580 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13581 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13582 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13583 }
13584 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13585
13586 /* add base */
13587 switch (bSib & X86_SIB_BASE_MASK)
13588 {
13589 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
13590 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
13591 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
13592 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
13593 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
13594 case 5:
13595 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13596 {
13597 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
13598 SET_SS_DEF();
13599 }
13600 else
13601 {
13602 uint32_t u32Disp;
13603 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13604 u32EffAddr += u32Disp;
13605 }
13606 break;
13607 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13608 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13609 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13610 }
13611 break;
13612 }
13613 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13614 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13615 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13616 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13617 }
13618
13619 /* Get and add the displacement. */
13620 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13621 {
13622 case 0:
13623 break;
13624 case 1:
13625 {
13626 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13627 u32EffAddr += i8Disp;
13628 break;
13629 }
13630 case 2:
13631 {
13632 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13633 u32EffAddr += u32Disp;
13634 break;
13635 }
13636 default:
13637 AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); /* (caller checked for these) */
13638 }
13639 }
13640
13641 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13642 {
13643 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
13644 return u32EffAddr;
13645 }
13646 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13647 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
13648 return u32EffAddr & UINT16_MAX;
13649 }
13650
13651 uint64_t u64EffAddr;
13652
13653 /* Handle the rip+disp32 form with no registers first. */
13654 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13655 {
13656 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13657 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13658 }
13659 else
13660 {
13661 /* Get the register (or SIB) value. */
13662 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13663 {
13664 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13665 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13666 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13667 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13668 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13669 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13670 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13671 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13672 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13673 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13674 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13675 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13676 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13677 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13678 /* SIB */
13679 case 4:
13680 case 12:
13681 {
13682 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13683
13684 /* Get the index and scale it. */
13685 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13686 {
13687 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13688 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13689 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13690 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13691 case 4: u64EffAddr = 0; /*none */ break;
13692 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13693 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13694 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13695 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13696 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13697 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13698 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13699 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13700 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13701 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13702 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13703 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13704 }
13705 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13706
13707 /* add base */
13708 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13709 {
13710 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13711 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13712 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13713 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13714 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
13715 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13716 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13717 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13718 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13719 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13720 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13721 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13722 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13723 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13724 /* complicated encodings */
13725 case 5:
13726 case 13:
13727 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13728 {
13729 if (!pVCpu->iem.s.uRexB)
13730 {
13731 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13732 SET_SS_DEF();
13733 }
13734 else
13735 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13736 }
13737 else
13738 {
13739 uint32_t u32Disp;
13740 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13741 u64EffAddr += (int32_t)u32Disp;
13742 }
13743 break;
13744 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13745 }
13746 break;
13747 }
13748 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13749 }
13750
13751 /* Get and add the displacement. */
13752 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13753 {
13754 case 0:
13755 break;
13756 case 1:
13757 {
13758 int8_t i8Disp;
13759 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13760 u64EffAddr += i8Disp;
13761 break;
13762 }
13763 case 2:
13764 {
13765 uint32_t u32Disp;
13766 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13767 u64EffAddr += (int32_t)u32Disp;
13768 break;
13769 }
13770 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
13771 }
13772
13773 }
13774
13775 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13776 {
13777 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
13778 return u64EffAddr;
13779 }
13780 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13781 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
13782 return u64EffAddr & UINT32_MAX;
13783}
13784#endif /* IEM_WITH_SETJMP */
13785
13786/** @} */
13787
13788
13789
13790/*
13791 * Include the instructions
13792 */
13793#include "IEMAllInstructions.cpp.h"
13794
13795
13796
13797#ifdef LOG_ENABLED
13798/**
13799 * Logs the current instruction.
13800 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
13801 * @param fSameCtx Set if we have the same context information as the VMM,
13802 * clear if we may have already executed an instruction in
13803 * our debug context. When clear, we assume IEMCPU holds
13804 * valid CPU mode info.
13805 *
13806 * The @a fSameCtx parameter is now misleading and obsolete.
13807 * @param pszFunction The IEM function doing the execution.
13808 */
13809IEM_STATIC void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction)
13810{
13811# ifdef IN_RING3
13812 if (LogIs2Enabled())
13813 {
13814 char szInstr[256];
13815 uint32_t cbInstr = 0;
13816 if (fSameCtx)
13817 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
13818 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
13819 szInstr, sizeof(szInstr), &cbInstr);
13820 else
13821 {
13822 uint32_t fFlags = 0;
13823 switch (pVCpu->iem.s.enmCpuMode)
13824 {
13825 case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;
13826 case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;
13827 case IEMMODE_16BIT:
13828 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
13829 fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
13830 else
13831 fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;
13832 break;
13833 }
13834 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
13835 szInstr, sizeof(szInstr), &cbInstr);
13836 }
13837
13838 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
13839 Log2(("**** %s\n"
13840 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
13841 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
13842 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
13843 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
13844 " %s\n"
13845 , pszFunction,
13846 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
13847 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
13848 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
13849 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
13850 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
13851 szInstr));
13852
13853 if (LogIs3Enabled())
13854 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
13855 }
13856 else
13857# endif
13858 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
13859 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
13860 RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
13861}
13862#endif /* LOG_ENABLED */
13863
13864
13865#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13866/**
13867 * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
13868 * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
13869 *
13870 * @returns Modified rcStrict.
13871 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13872 * @param rcStrict The instruction execution status.
13873 */
13874static VBOXSTRICTRC iemHandleNestedInstructionBoundraryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
13875{
13876 Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
13877 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF))
13878 {
13879 /* VMX preemption timer takes priority over NMI-window exits. */
13880 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
13881 {
13882 rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
13883 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
13884 }
13885 /*
13886 * Check remaining intercepts.
13887 *
13888 * NMI-window and Interrupt-window VM-exits.
13889 * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
13890 * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
13891 *
13892 * See Intel spec. 26.7.6 "NMI-Window Exiting".
13893 * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
13894 */
13895 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW)
13896 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13897 && !TRPMHasTrap(pVCpu))
13898 {
13899 Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
13900 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
13901 && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
13902 {
13903 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
13904 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
13905 }
13906 else if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
13907 && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
13908 {
13909 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
13910 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
13911 }
13912 }
13913 }
13914 /* TPR-below threshold/APIC write has the highest priority. */
13915 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
13916 {
13917 rcStrict = iemVmxApicWriteEmulation(pVCpu);
13918 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13919 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
13920 }
13921 /* MTF takes priority over VMX-preemption timer. */
13922 else
13923 {
13924 rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
13925 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13926 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
13927 }
13928 return rcStrict;
13929}
13930#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
13931
13932
13933/**
13934 * Makes status code addjustments (pass up from I/O and access handler)
13935 * as well as maintaining statistics.
13936 *
13937 * @returns Strict VBox status code to pass up.
13938 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13939 * @param rcStrict The status from executing an instruction.
13940 */
13941DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
13942{
13943 if (rcStrict != VINF_SUCCESS)
13944 {
13945 if (RT_SUCCESS(rcStrict))
13946 {
13947 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
13948 || rcStrict == VINF_IOM_R3_IOPORT_READ
13949 || rcStrict == VINF_IOM_R3_IOPORT_WRITE
13950 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
13951 || rcStrict == VINF_IOM_R3_MMIO_READ
13952 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
13953 || rcStrict == VINF_IOM_R3_MMIO_WRITE
13954 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
13955 || rcStrict == VINF_CPUM_R3_MSR_READ
13956 || rcStrict == VINF_CPUM_R3_MSR_WRITE
13957 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
13958 || rcStrict == VINF_EM_RAW_TO_R3
13959 || rcStrict == VINF_EM_TRIPLE_FAULT
13960 || rcStrict == VINF_GIM_R3_HYPERCALL
13961 /* raw-mode / virt handlers only: */
13962 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
13963 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
13964 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
13965 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
13966 || rcStrict == VINF_SELM_SYNC_GDT
13967 || rcStrict == VINF_CSAM_PENDING_ACTION
13968 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
13969 /* nested hw.virt codes: */
13970 || rcStrict == VINF_VMX_VMEXIT
13971 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
13972 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
13973 || rcStrict == VINF_SVM_VMEXIT
13974 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13975/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
13976 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
13977#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
13978 if ( rcStrict == VINF_VMX_VMEXIT
13979 && rcPassUp == VINF_SUCCESS)
13980 rcStrict = VINF_SUCCESS;
13981 else
13982#endif
13983#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
13984 if ( rcStrict == VINF_SVM_VMEXIT
13985 && rcPassUp == VINF_SUCCESS)
13986 rcStrict = VINF_SUCCESS;
13987 else
13988#endif
13989 if (rcPassUp == VINF_SUCCESS)
13990 pVCpu->iem.s.cRetInfStatuses++;
13991 else if ( rcPassUp < VINF_EM_FIRST
13992 || rcPassUp > VINF_EM_LAST
13993 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
13994 {
13995 Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
13996 pVCpu->iem.s.cRetPassUpStatus++;
13997 rcStrict = rcPassUp;
13998 }
13999 else
14000 {
14001 Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
14002 pVCpu->iem.s.cRetInfStatuses++;
14003 }
14004 }
14005 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
14006 pVCpu->iem.s.cRetAspectNotImplemented++;
14007 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
14008 pVCpu->iem.s.cRetInstrNotImplemented++;
14009 else
14010 pVCpu->iem.s.cRetErrStatuses++;
14011 }
14012 else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
14013 {
14014 pVCpu->iem.s.cRetPassUpStatus++;
14015 rcStrict = pVCpu->iem.s.rcPassUp;
14016 }
14017
14018 return rcStrict;
14019}
14020
14021
14022/**
14023 * The actual code execution bits of IEMExecOne, IEMExecOneEx, and
14024 * IEMExecOneWithPrefetchedByPC.
14025 *
14026 * Similar code is found in IEMExecLots.
14027 *
14028 * @return Strict VBox status code.
14029 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14030 * @param fExecuteInhibit If set, execute the instruction following CLI,
14031 * POP SS and MOV SS,GR.
14032 * @param pszFunction The calling function name.
14033 */
14034DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
14035{
14036 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
14037 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
14038 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
14039 RT_NOREF_PV(pszFunction);
14040
14041#ifdef IEM_WITH_SETJMP
14042 VBOXSTRICTRC rcStrict;
14043 jmp_buf JmpBuf;
14044 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14045 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14046 if ((rcStrict = setjmp(JmpBuf)) == 0)
14047 {
14048 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14049 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14050 }
14051 else
14052 pVCpu->iem.s.cLongJumps++;
14053 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14054#else
14055 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14056 VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14057#endif
14058 if (rcStrict == VINF_SUCCESS)
14059 pVCpu->iem.s.cInstructions++;
14060 if (pVCpu->iem.s.cActiveMappings > 0)
14061 {
14062 Assert(rcStrict != VINF_SUCCESS);
14063 iemMemRollback(pVCpu);
14064 }
14065 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
14066 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
14067 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
14068
14069//#ifdef DEBUG
14070// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
14071//#endif
14072
14073#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
14074 /*
14075 * Perform any VMX nested-guest instruction boundary actions.
14076 *
14077 * If any of these causes a VM-exit, we must skip executing the next
14078 * instruction (would run into stale page tables). A VM-exit makes sure
14079 * there is no interrupt-inhibition, so that should ensure we don't go
14080 * to try execute the next instruction. Clearing fExecuteInhibit is
14081 * problematic because of the setjmp/longjmp clobbering above.
14082 */
14083 if ( rcStrict == VINF_SUCCESS
14084 && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
14085 | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))
14086 rcStrict = iemHandleNestedInstructionBoundraryFFs(pVCpu, rcStrict);
14087#endif
14088
14089 /* Execute the next instruction as well if a cli, pop ss or
14090 mov ss, Gr has just completed successfully. */
14091 if ( fExecuteInhibit
14092 && rcStrict == VINF_SUCCESS
14093 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
14094 && EMIsInhibitInterruptsActive(pVCpu))
14095 {
14096 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock);
14097 if (rcStrict == VINF_SUCCESS)
14098 {
14099#ifdef LOG_ENABLED
14100 iemLogCurInstr(pVCpu, false, pszFunction);
14101#endif
14102#ifdef IEM_WITH_SETJMP
14103 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14104 if ((rcStrict = setjmp(JmpBuf)) == 0)
14105 {
14106 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14107 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14108 }
14109 else
14110 pVCpu->iem.s.cLongJumps++;
14111 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14112#else
14113 IEM_OPCODE_GET_NEXT_U8(&b);
14114 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14115#endif
14116 if (rcStrict == VINF_SUCCESS)
14117 pVCpu->iem.s.cInstructions++;
14118 if (pVCpu->iem.s.cActiveMappings > 0)
14119 {
14120 Assert(rcStrict != VINF_SUCCESS);
14121 iemMemRollback(pVCpu);
14122 }
14123 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
14124 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
14125 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
14126 }
14127 else if (pVCpu->iem.s.cActiveMappings > 0)
14128 iemMemRollback(pVCpu);
14129 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); /* hope this is correct for all exceptional cases... */
14130 }
14131
14132 /*
14133 * Return value fiddling, statistics and sanity assertions.
14134 */
14135 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14136
14137 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14138 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14139 return rcStrict;
14140}
14141
14142
14143/**
14144 * Execute one instruction.
14145 *
14146 * @return Strict VBox status code.
14147 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14148 */
14149VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
14150{
14151 AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
14152#ifdef LOG_ENABLED
14153 iemLogCurInstr(pVCpu, true, "IEMExecOne");
14154#endif
14155
14156 /*
14157 * Do the decoding and emulation.
14158 */
14159 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
14160 if (rcStrict == VINF_SUCCESS)
14161 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
14162 else if (pVCpu->iem.s.cActiveMappings > 0)
14163 iemMemRollback(pVCpu);
14164
14165 if (rcStrict != VINF_SUCCESS)
14166 LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14167 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14168 return rcStrict;
14169}
14170
14171
14172VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
14173{
14174 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14175
14176 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14177 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
14178 if (rcStrict == VINF_SUCCESS)
14179 {
14180 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
14181 if (pcbWritten)
14182 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14183 }
14184 else if (pVCpu->iem.s.cActiveMappings > 0)
14185 iemMemRollback(pVCpu);
14186
14187 return rcStrict;
14188}
14189
14190
14191VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14192 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
14193{
14194 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14195
14196 VBOXSTRICTRC rcStrict;
14197 if ( cbOpcodeBytes
14198 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14199 {
14200 iemInitDecoder(pVCpu, false, false);
14201#ifdef IEM_WITH_CODE_TLB
14202 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14203 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14204 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14205 pVCpu->iem.s.offCurInstrStart = 0;
14206 pVCpu->iem.s.offInstrNextByte = 0;
14207#else
14208 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14209 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14210#endif
14211 rcStrict = VINF_SUCCESS;
14212 }
14213 else
14214 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
14215 if (rcStrict == VINF_SUCCESS)
14216 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
14217 else if (pVCpu->iem.s.cActiveMappings > 0)
14218 iemMemRollback(pVCpu);
14219
14220 return rcStrict;
14221}
14222
14223
14224VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
14225{
14226 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14227
14228 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14229 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
14230 if (rcStrict == VINF_SUCCESS)
14231 {
14232 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
14233 if (pcbWritten)
14234 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14235 }
14236 else if (pVCpu->iem.s.cActiveMappings > 0)
14237 iemMemRollback(pVCpu);
14238
14239 return rcStrict;
14240}
14241
14242
14243VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14244 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
14245{
14246 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14247
14248 VBOXSTRICTRC rcStrict;
14249 if ( cbOpcodeBytes
14250 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14251 {
14252 iemInitDecoder(pVCpu, true, false);
14253#ifdef IEM_WITH_CODE_TLB
14254 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14255 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14256 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14257 pVCpu->iem.s.offCurInstrStart = 0;
14258 pVCpu->iem.s.offInstrNextByte = 0;
14259#else
14260 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14261 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14262#endif
14263 rcStrict = VINF_SUCCESS;
14264 }
14265 else
14266 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
14267 if (rcStrict == VINF_SUCCESS)
14268 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
14269 else if (pVCpu->iem.s.cActiveMappings > 0)
14270 iemMemRollback(pVCpu);
14271
14272 return rcStrict;
14273}
14274
14275
14276/**
14277 * For debugging DISGetParamSize, may come in handy.
14278 *
14279 * @returns Strict VBox status code.
14280 * @param pVCpu The cross context virtual CPU structure of the
14281 * calling EMT.
14282 * @param pCtxCore The context core structure.
14283 * @param OpcodeBytesPC The PC of the opcode bytes.
14284 * @param pvOpcodeBytes Prefeched opcode bytes.
14285 * @param cbOpcodeBytes Number of prefetched bytes.
14286 * @param pcbWritten Where to return the number of bytes written.
14287 * Optional.
14288 */
14289VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14290 const void *pvOpcodeBytes, size_t cbOpcodeBytes,
14291 uint32_t *pcbWritten)
14292{
14293 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14294
14295 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14296 VBOXSTRICTRC rcStrict;
14297 if ( cbOpcodeBytes
14298 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14299 {
14300 iemInitDecoder(pVCpu, true, false);
14301#ifdef IEM_WITH_CODE_TLB
14302 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14303 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14304 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14305 pVCpu->iem.s.offCurInstrStart = 0;
14306 pVCpu->iem.s.offInstrNextByte = 0;
14307#else
14308 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14309 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14310#endif
14311 rcStrict = VINF_SUCCESS;
14312 }
14313 else
14314 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
14315 if (rcStrict == VINF_SUCCESS)
14316 {
14317 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
14318 if (pcbWritten)
14319 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14320 }
14321 else if (pVCpu->iem.s.cActiveMappings > 0)
14322 iemMemRollback(pVCpu);
14323
14324 return rcStrict;
14325}
14326
14327
14328/**
14329 * For handling split cacheline lock operations when the host has split-lock
14330 * detection enabled.
14331 *
14332 * This will cause the interpreter to disregard the lock prefix and implicit
14333 * locking (xchg).
14334 *
14335 * @returns Strict VBox status code.
14336 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14337 */
14338VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
14339{
14340 /*
14341 * Do the decoding and emulation.
14342 */
14343 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /*fDisregardLock*/);
14344 if (rcStrict == VINF_SUCCESS)
14345 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
14346 else if (pVCpu->iem.s.cActiveMappings > 0)
14347 iemMemRollback(pVCpu);
14348
14349 if (rcStrict != VINF_SUCCESS)
14350 LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14351 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14352 return rcStrict;
14353}
14354
14355
14356VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
14357{
14358 uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
14359 AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
14360
14361 /*
14362 * See if there is an interrupt pending in TRPM, inject it if we can.
14363 */
14364 /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
14365#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14366 bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
14367 if (fIntrEnabled)
14368 {
14369 if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
14370 fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
14371 else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
14372 fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
14373 else
14374 {
14375 Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
14376 fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
14377 }
14378 }
14379#else
14380 bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
14381#endif
14382
14383 /** @todo What if we are injecting an exception and not an interrupt? Is that
14384 * possible here? For now we assert it is indeed only an interrupt. */
14385 if ( fIntrEnabled
14386 && TRPMHasTrap(pVCpu)
14387 && EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
14388 {
14389 uint8_t u8TrapNo;
14390 TRPMEVENT enmType;
14391 uint32_t uErrCode;
14392 RTGCPTR uCr2;
14393 int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */, NULL /* fIcebp */);
14394 AssertRC(rc2);
14395 Assert(enmType == TRPM_HARDWARE_INT);
14396 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
14397 TRPMResetTrap(pVCpu);
14398#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14399 /* Injecting an event may cause a VM-exit. */
14400 if ( rcStrict != VINF_SUCCESS
14401 && rcStrict != VINF_IEM_RAISED_XCPT)
14402 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
14403#else
14404 NOREF(rcStrict);
14405#endif
14406 }
14407
14408 /*
14409 * Initial decoder init w/ prefetch, then setup setjmp.
14410 */
14411 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
14412 if (rcStrict == VINF_SUCCESS)
14413 {
14414#ifdef IEM_WITH_SETJMP
14415 jmp_buf JmpBuf;
14416 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14417 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14418 pVCpu->iem.s.cActiveMappings = 0;
14419 if ((rcStrict = setjmp(JmpBuf)) == 0)
14420#endif
14421 {
14422 /*
14423 * The run loop. We limit ourselves to 4096 instructions right now.
14424 */
14425 uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
14426 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
14427 for (;;)
14428 {
14429 /*
14430 * Log the state.
14431 */
14432#ifdef LOG_ENABLED
14433 iemLogCurInstr(pVCpu, true, "IEMExecLots");
14434#endif
14435
14436 /*
14437 * Do the decoding and emulation.
14438 */
14439 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14440 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14441 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14442 {
14443 Assert(pVCpu->iem.s.cActiveMappings == 0);
14444 pVCpu->iem.s.cInstructions++;
14445 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
14446 {
14447 uint64_t fCpu = pVCpu->fLocalForcedActions
14448 & ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
14449 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
14450 | VMCPU_FF_TLB_FLUSH
14451 | VMCPU_FF_INHIBIT_INTERRUPTS
14452 | VMCPU_FF_BLOCK_NMIS
14453 | VMCPU_FF_UNHALT ));
14454
14455 if (RT_LIKELY( ( !fCpu
14456 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
14457 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
14458 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
14459 {
14460 if (cMaxInstructionsGccStupidity-- > 0)
14461 {
14462 /* Poll timers every now an then according to the caller's specs. */
14463 if ( (cMaxInstructionsGccStupidity & cPollRate) != 0
14464 || !TMTimerPollBool(pVM, pVCpu))
14465 {
14466 Assert(pVCpu->iem.s.cActiveMappings == 0);
14467 iemReInitDecoder(pVCpu);
14468 continue;
14469 }
14470 }
14471 }
14472 }
14473 Assert(pVCpu->iem.s.cActiveMappings == 0);
14474 }
14475 else if (pVCpu->iem.s.cActiveMappings > 0)
14476 iemMemRollback(pVCpu);
14477 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14478 break;
14479 }
14480 }
14481#ifdef IEM_WITH_SETJMP
14482 else
14483 {
14484 if (pVCpu->iem.s.cActiveMappings > 0)
14485 iemMemRollback(pVCpu);
14486# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14487 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14488# endif
14489 pVCpu->iem.s.cLongJumps++;
14490 }
14491 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14492#endif
14493
14494 /*
14495 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
14496 */
14497 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14498 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14499 }
14500 else
14501 {
14502 if (pVCpu->iem.s.cActiveMappings > 0)
14503 iemMemRollback(pVCpu);
14504
14505#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14506 /*
14507 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
14508 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
14509 */
14510 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14511#endif
14512 }
14513
14514 /*
14515 * Maybe re-enter raw-mode and log.
14516 */
14517 if (rcStrict != VINF_SUCCESS)
14518 LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14519 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14520 if (pcInstructions)
14521 *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
14522 return rcStrict;
14523}
14524
14525
14526/**
14527 * Interface used by EMExecuteExec, does exit statistics and limits.
14528 *
14529 * @returns Strict VBox status code.
14530 * @param pVCpu The cross context virtual CPU structure.
14531 * @param fWillExit To be defined.
14532 * @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
14533 * @param cMaxInstructions Maximum number of instructions to execute.
14534 * @param cMaxInstructionsWithoutExits
14535 * The max number of instructions without exits.
14536 * @param pStats Where to return statistics.
14537 */
14538VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
14539 uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
14540{
14541 NOREF(fWillExit); /** @todo define flexible exit crits */
14542
14543 /*
14544 * Initialize return stats.
14545 */
14546 pStats->cInstructions = 0;
14547 pStats->cExits = 0;
14548 pStats->cMaxExitDistance = 0;
14549 pStats->cReserved = 0;
14550
14551 /*
14552 * Initial decoder init w/ prefetch, then setup setjmp.
14553 */
14554 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
14555 if (rcStrict == VINF_SUCCESS)
14556 {
14557#ifdef IEM_WITH_SETJMP
14558 jmp_buf JmpBuf;
14559 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14560 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14561 pVCpu->iem.s.cActiveMappings = 0;
14562 if ((rcStrict = setjmp(JmpBuf)) == 0)
14563#endif
14564 {
14565#ifdef IN_RING0
14566 bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
14567#endif
14568 uint32_t cInstructionSinceLastExit = 0;
14569
14570 /*
14571 * The run loop. We limit ourselves to 4096 instructions right now.
14572 */
14573 PVM pVM = pVCpu->CTX_SUFF(pVM);
14574 for (;;)
14575 {
14576 /*
14577 * Log the state.
14578 */
14579#ifdef LOG_ENABLED
14580 iemLogCurInstr(pVCpu, true, "IEMExecForExits");
14581#endif
14582
14583 /*
14584 * Do the decoding and emulation.
14585 */
14586 uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
14587
14588 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14589 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14590
14591 if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
14592 && cInstructionSinceLastExit > 0 /* don't count the first */ )
14593 {
14594 pStats->cExits += 1;
14595 if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
14596 pStats->cMaxExitDistance = cInstructionSinceLastExit;
14597 cInstructionSinceLastExit = 0;
14598 }
14599
14600 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14601 {
14602 Assert(pVCpu->iem.s.cActiveMappings == 0);
14603 pVCpu->iem.s.cInstructions++;
14604 pStats->cInstructions++;
14605 cInstructionSinceLastExit++;
14606 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
14607 {
14608 uint64_t fCpu = pVCpu->fLocalForcedActions
14609 & ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
14610 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
14611 | VMCPU_FF_TLB_FLUSH
14612 | VMCPU_FF_INHIBIT_INTERRUPTS
14613 | VMCPU_FF_BLOCK_NMIS
14614 | VMCPU_FF_UNHALT ));
14615
14616 if (RT_LIKELY( ( ( !fCpu
14617 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
14618 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
14619 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
14620 || pStats->cInstructions < cMinInstructions))
14621 {
14622 if (pStats->cInstructions < cMaxInstructions)
14623 {
14624 if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
14625 {
14626#ifdef IN_RING0
14627 if ( !fCheckPreemptionPending
14628 || !RTThreadPreemptIsPending(NIL_RTTHREAD))
14629#endif
14630 {
14631 Assert(pVCpu->iem.s.cActiveMappings == 0);
14632 iemReInitDecoder(pVCpu);
14633 continue;
14634 }
14635#ifdef IN_RING0
14636 rcStrict = VINF_EM_RAW_INTERRUPT;
14637 break;
14638#endif
14639 }
14640 }
14641 }
14642 Assert(!(fCpu & VMCPU_FF_IEM));
14643 }
14644 Assert(pVCpu->iem.s.cActiveMappings == 0);
14645 }
14646 else if (pVCpu->iem.s.cActiveMappings > 0)
14647 iemMemRollback(pVCpu);
14648 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14649 break;
14650 }
14651 }
14652#ifdef IEM_WITH_SETJMP
14653 else
14654 {
14655 if (pVCpu->iem.s.cActiveMappings > 0)
14656 iemMemRollback(pVCpu);
14657 pVCpu->iem.s.cLongJumps++;
14658 }
14659 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14660#endif
14661
14662 /*
14663 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
14664 */
14665 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14666 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14667 }
14668 else
14669 {
14670 if (pVCpu->iem.s.cActiveMappings > 0)
14671 iemMemRollback(pVCpu);
14672
14673#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
14674 /*
14675 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
14676 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
14677 */
14678 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14679#endif
14680 }
14681
14682 /*
14683 * Maybe re-enter raw-mode and log.
14684 */
14685 if (rcStrict != VINF_SUCCESS)
14686 LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
14687 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
14688 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
14689 return rcStrict;
14690}
14691
14692
14693/**
14694 * Injects a trap, fault, abort, software interrupt or external interrupt.
14695 *
14696 * The parameter list matches TRPMQueryTrapAll pretty closely.
14697 *
14698 * @returns Strict VBox status code.
14699 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14700 * @param u8TrapNo The trap number.
14701 * @param enmType What type is it (trap/fault/abort), software
14702 * interrupt or hardware interrupt.
14703 * @param uErrCode The error code if applicable.
14704 * @param uCr2 The CR2 value if applicable.
14705 * @param cbInstr The instruction length (only relevant for
14706 * software interrupts).
14707 */
14708VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
14709 uint8_t cbInstr)
14710{
14711 iemInitDecoder(pVCpu, false, false);
14712#ifdef DBGFTRACE_ENABLED
14713 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
14714 u8TrapNo, enmType, uErrCode, uCr2);
14715#endif
14716
14717 uint32_t fFlags;
14718 switch (enmType)
14719 {
14720 case TRPM_HARDWARE_INT:
14721 Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
14722 fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
14723 uErrCode = uCr2 = 0;
14724 break;
14725
14726 case TRPM_SOFTWARE_INT:
14727 Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
14728 fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
14729 uErrCode = uCr2 = 0;
14730 break;
14731
14732 case TRPM_TRAP:
14733 Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
14734 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
14735 if (u8TrapNo == X86_XCPT_PF)
14736 fFlags |= IEM_XCPT_FLAGS_CR2;
14737 switch (u8TrapNo)
14738 {
14739 case X86_XCPT_DF:
14740 case X86_XCPT_TS:
14741 case X86_XCPT_NP:
14742 case X86_XCPT_SS:
14743 case X86_XCPT_PF:
14744 case X86_XCPT_AC:
14745 case X86_XCPT_GP:
14746 fFlags |= IEM_XCPT_FLAGS_ERR;
14747 break;
14748 }
14749 break;
14750
14751 IEM_NOT_REACHED_DEFAULT_CASE_RET();
14752 }
14753
14754 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
14755
14756 if (pVCpu->iem.s.cActiveMappings > 0)
14757 iemMemRollback(pVCpu);
14758
14759 return rcStrict;
14760}
14761
14762
14763/**
14764 * Injects the active TRPM event.
14765 *
14766 * @returns Strict VBox status code.
14767 * @param pVCpu The cross context virtual CPU structure.
14768 */
14769VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
14770{
14771#ifndef IEM_IMPLEMENTS_TASKSWITCH
14772 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
14773#else
14774 uint8_t u8TrapNo;
14775 TRPMEVENT enmType;
14776 uint32_t uErrCode;
14777 RTGCUINTPTR uCr2;
14778 uint8_t cbInstr;
14779 int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
14780 if (RT_FAILURE(rc))
14781 return rc;
14782
14783 /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
14784 * ICEBP \#DB injection as a special case. */
14785 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
14786#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
14787 if (rcStrict == VINF_SVM_VMEXIT)
14788 rcStrict = VINF_SUCCESS;
14789#endif
14790#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
14791 if (rcStrict == VINF_VMX_VMEXIT)
14792 rcStrict = VINF_SUCCESS;
14793#endif
14794 /** @todo Are there any other codes that imply the event was successfully
14795 * delivered to the guest? See @bugref{6607}. */
14796 if ( rcStrict == VINF_SUCCESS
14797 || rcStrict == VINF_IEM_RAISED_XCPT)
14798 TRPMResetTrap(pVCpu);
14799
14800 return rcStrict;
14801#endif
14802}
14803
14804
14805VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
14806{
14807 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
14808 return VERR_NOT_IMPLEMENTED;
14809}
14810
14811
14812VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
14813{
14814 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
14815 return VERR_NOT_IMPLEMENTED;
14816}
14817
14818
14819#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
14820/**
14821 * Executes a IRET instruction with default operand size.
14822 *
14823 * This is for PATM.
14824 *
14825 * @returns VBox status code.
14826 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14827 * @param pCtxCore The register frame.
14828 */
14829VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPUCC pVCpu, PCPUMCTXCORE pCtxCore)
14830{
14831 PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
14832
14833 iemCtxCoreToCtx(pCtx, pCtxCore);
14834 iemInitDecoder(pVCpu);
14835 VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
14836 if (rcStrict == VINF_SUCCESS)
14837 iemCtxToCtxCore(pCtxCore, pCtx);
14838 else
14839 LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14840 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14841 return rcStrict;
14842}
14843#endif
14844
14845
14846/**
14847 * Macro used by the IEMExec* method to check the given instruction length.
14848 *
14849 * Will return on failure!
14850 *
14851 * @param a_cbInstr The given instruction length.
14852 * @param a_cbMin The minimum length.
14853 */
14854#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
14855 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
14856 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
14857
14858
14859/**
14860 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
14861 *
14862 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
14863 *
14864 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
14865 * @param pVCpu The cross context virtual CPU structure of the calling thread.
14866 * @param rcStrict The status code to fiddle.
14867 */
14868DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
14869{
14870 iemUninitExec(pVCpu);
14871 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
14872}
14873
14874
14875/**
14876 * Interface for HM and EM for executing string I/O OUT (write) instructions.
14877 *
14878 * This API ASSUMES that the caller has already verified that the guest code is
14879 * allowed to access the I/O port. (The I/O port is in the DX register in the
14880 * guest state.)
14881 *
14882 * @returns Strict VBox status code.
14883 * @param pVCpu The cross context virtual CPU structure.
14884 * @param cbValue The size of the I/O port access (1, 2, or 4).
14885 * @param enmAddrMode The addressing mode.
14886 * @param fRepPrefix Indicates whether a repeat prefix is used
14887 * (doesn't matter which for this instruction).
14888 * @param cbInstr The instruction length in bytes.
14889 * @param iEffSeg The effective segment address.
14890 * @param fIoChecked Whether the access to the I/O port has been
14891 * checked or not. It's typically checked in the
14892 * HM scenario.
14893 */
14894VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
14895 bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
14896{
14897 AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
14898 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14899
14900 /*
14901 * State init.
14902 */
14903 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14904
14905 /*
14906 * Switch orgy for getting to the right handler.
14907 */
14908 VBOXSTRICTRC rcStrict;
14909 if (fRepPrefix)
14910 {
14911 switch (enmAddrMode)
14912 {
14913 case IEMMODE_16BIT:
14914 switch (cbValue)
14915 {
14916 case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14917 case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14918 case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14919 default:
14920 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14921 }
14922 break;
14923
14924 case IEMMODE_32BIT:
14925 switch (cbValue)
14926 {
14927 case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14928 case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14929 case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14930 default:
14931 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14932 }
14933 break;
14934
14935 case IEMMODE_64BIT:
14936 switch (cbValue)
14937 {
14938 case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14939 case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14940 case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14941 default:
14942 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14943 }
14944 break;
14945
14946 default:
14947 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14948 }
14949 }
14950 else
14951 {
14952 switch (enmAddrMode)
14953 {
14954 case IEMMODE_16BIT:
14955 switch (cbValue)
14956 {
14957 case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14958 case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14959 case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14960 default:
14961 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14962 }
14963 break;
14964
14965 case IEMMODE_32BIT:
14966 switch (cbValue)
14967 {
14968 case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14969 case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14970 case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14971 default:
14972 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14973 }
14974 break;
14975
14976 case IEMMODE_64BIT:
14977 switch (cbValue)
14978 {
14979 case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14980 case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14981 case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14982 default:
14983 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14984 }
14985 break;
14986
14987 default:
14988 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14989 }
14990 }
14991
14992 if (pVCpu->iem.s.cActiveMappings)
14993 iemMemRollback(pVCpu);
14994
14995 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14996}
14997
14998
14999/**
15000 * Interface for HM and EM for executing string I/O IN (read) instructions.
15001 *
15002 * This API ASSUMES that the caller has already verified that the guest code is
15003 * allowed to access the I/O port. (The I/O port is in the DX register in the
15004 * guest state.)
15005 *
15006 * @returns Strict VBox status code.
15007 * @param pVCpu The cross context virtual CPU structure.
15008 * @param cbValue The size of the I/O port access (1, 2, or 4).
15009 * @param enmAddrMode The addressing mode.
15010 * @param fRepPrefix Indicates whether a repeat prefix is used
15011 * (doesn't matter which for this instruction).
15012 * @param cbInstr The instruction length in bytes.
15013 * @param fIoChecked Whether the access to the I/O port has been
15014 * checked or not. It's typically checked in the
15015 * HM scenario.
15016 */
15017VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
15018 bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
15019{
15020 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15021
15022 /*
15023 * State init.
15024 */
15025 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15026
15027 /*
15028 * Switch orgy for getting to the right handler.
15029 */
15030 VBOXSTRICTRC rcStrict;
15031 if (fRepPrefix)
15032 {
15033 switch (enmAddrMode)
15034 {
15035 case IEMMODE_16BIT:
15036 switch (cbValue)
15037 {
15038 case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
15039 case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
15040 case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
15041 default:
15042 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15043 }
15044 break;
15045
15046 case IEMMODE_32BIT:
15047 switch (cbValue)
15048 {
15049 case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
15050 case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
15051 case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
15052 default:
15053 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15054 }
15055 break;
15056
15057 case IEMMODE_64BIT:
15058 switch (cbValue)
15059 {
15060 case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
15061 case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
15062 case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
15063 default:
15064 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15065 }
15066 break;
15067
15068 default:
15069 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
15070 }
15071 }
15072 else
15073 {
15074 switch (enmAddrMode)
15075 {
15076 case IEMMODE_16BIT:
15077 switch (cbValue)
15078 {
15079 case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
15080 case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
15081 case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
15082 default:
15083 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15084 }
15085 break;
15086
15087 case IEMMODE_32BIT:
15088 switch (cbValue)
15089 {
15090 case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
15091 case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
15092 case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
15093 default:
15094 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15095 }
15096 break;
15097
15098 case IEMMODE_64BIT:
15099 switch (cbValue)
15100 {
15101 case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
15102 case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
15103 case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
15104 default:
15105 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
15106 }
15107 break;
15108
15109 default:
15110 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
15111 }
15112 }
15113
15114 if ( pVCpu->iem.s.cActiveMappings == 0
15115 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
15116 { /* likely */ }
15117 else
15118 {
15119 AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
15120 iemMemRollback(pVCpu);
15121 }
15122 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15123}
15124
15125
15126/**
15127 * Interface for rawmode to write execute an OUT instruction.
15128 *
15129 * @returns Strict VBox status code.
15130 * @param pVCpu The cross context virtual CPU structure.
15131 * @param cbInstr The instruction length in bytes.
15132 * @param u16Port The port to read.
15133 * @param fImm Whether the port is specified using an immediate operand or
15134 * using the implicit DX register.
15135 * @param cbReg The register size.
15136 *
15137 * @remarks In ring-0 not all of the state needs to be synced in.
15138 */
15139VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
15140{
15141 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15142 Assert(cbReg <= 4 && cbReg != 3);
15143
15144 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15145 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
15146 Assert(!pVCpu->iem.s.cActiveMappings);
15147 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15148}
15149
15150
15151/**
15152 * Interface for rawmode to write execute an IN instruction.
15153 *
15154 * @returns Strict VBox status code.
15155 * @param pVCpu The cross context virtual CPU structure.
15156 * @param cbInstr The instruction length in bytes.
15157 * @param u16Port The port to read.
15158 * @param fImm Whether the port is specified using an immediate operand or
15159 * using the implicit DX.
15160 * @param cbReg The register size.
15161 */
15162VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
15163{
15164 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15165 Assert(cbReg <= 4 && cbReg != 3);
15166
15167 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15168 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
15169 Assert(!pVCpu->iem.s.cActiveMappings);
15170 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15171}
15172
15173
15174/**
15175 * Interface for HM and EM to write to a CRx register.
15176 *
15177 * @returns Strict VBox status code.
15178 * @param pVCpu The cross context virtual CPU structure.
15179 * @param cbInstr The instruction length in bytes.
15180 * @param iCrReg The control register number (destination).
15181 * @param iGReg The general purpose register number (source).
15182 *
15183 * @remarks In ring-0 not all of the state needs to be synced in.
15184 */
15185VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
15186{
15187 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15188 Assert(iCrReg < 16);
15189 Assert(iGReg < 16);
15190
15191 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15192 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
15193 Assert(!pVCpu->iem.s.cActiveMappings);
15194 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15195}
15196
15197
15198/**
15199 * Interface for HM and EM to read from a CRx register.
15200 *
15201 * @returns Strict VBox status code.
15202 * @param pVCpu The cross context virtual CPU structure.
15203 * @param cbInstr The instruction length in bytes.
15204 * @param iGReg The general purpose register number (destination).
15205 * @param iCrReg The control register number (source).
15206 *
15207 * @remarks In ring-0 not all of the state needs to be synced in.
15208 */
15209VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
15210{
15211 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15212 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
15213 | CPUMCTX_EXTRN_APIC_TPR);
15214 Assert(iCrReg < 16);
15215 Assert(iGReg < 16);
15216
15217 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15218 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
15219 Assert(!pVCpu->iem.s.cActiveMappings);
15220 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15221}
15222
15223
15224/**
15225 * Interface for HM and EM to clear the CR0[TS] bit.
15226 *
15227 * @returns Strict VBox status code.
15228 * @param pVCpu The cross context virtual CPU structure.
15229 * @param cbInstr The instruction length in bytes.
15230 *
15231 * @remarks In ring-0 not all of the state needs to be synced in.
15232 */
15233VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
15234{
15235 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15236
15237 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15238 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
15239 Assert(!pVCpu->iem.s.cActiveMappings);
15240 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15241}
15242
15243
15244/**
15245 * Interface for HM and EM to emulate the LMSW instruction (loads CR0).
15246 *
15247 * @returns Strict VBox status code.
15248 * @param pVCpu The cross context virtual CPU structure.
15249 * @param cbInstr The instruction length in bytes.
15250 * @param uValue The value to load into CR0.
15251 * @param GCPtrEffDst The guest-linear address if the LMSW instruction has a
15252 * memory operand. Otherwise pass NIL_RTGCPTR.
15253 *
15254 * @remarks In ring-0 not all of the state needs to be synced in.
15255 */
15256VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
15257{
15258 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15259
15260 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15261 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
15262 Assert(!pVCpu->iem.s.cActiveMappings);
15263 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15264}
15265
15266
15267/**
15268 * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
15269 *
15270 * Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
15271 *
15272 * @returns Strict VBox status code.
15273 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15274 * @param cbInstr The instruction length in bytes.
15275 * @remarks In ring-0 not all of the state needs to be synced in.
15276 * @thread EMT(pVCpu)
15277 */
15278VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
15279{
15280 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15281
15282 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15283 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
15284 Assert(!pVCpu->iem.s.cActiveMappings);
15285 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15286}
15287
15288
15289/**
15290 * Interface for HM and EM to emulate the WBINVD instruction.
15291 *
15292 * @returns Strict VBox status code.
15293 * @param pVCpu The cross context virtual CPU structure.
15294 * @param cbInstr The instruction length in bytes.
15295 *
15296 * @remarks In ring-0 not all of the state needs to be synced in.
15297 */
15298VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
15299{
15300 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15301
15302 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15303 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
15304 Assert(!pVCpu->iem.s.cActiveMappings);
15305 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15306}
15307
15308
15309/**
15310 * Interface for HM and EM to emulate the INVD instruction.
15311 *
15312 * @returns Strict VBox status code.
15313 * @param pVCpu The cross context virtual CPU structure.
15314 * @param cbInstr The instruction length in bytes.
15315 *
15316 * @remarks In ring-0 not all of the state needs to be synced in.
15317 */
15318VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
15319{
15320 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15321
15322 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15323 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
15324 Assert(!pVCpu->iem.s.cActiveMappings);
15325 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15326}
15327
15328
15329/**
15330 * Interface for HM and EM to emulate the INVLPG instruction.
15331 *
15332 * @returns Strict VBox status code.
15333 * @retval VINF_PGM_SYNC_CR3
15334 *
15335 * @param pVCpu The cross context virtual CPU structure.
15336 * @param cbInstr The instruction length in bytes.
15337 * @param GCPtrPage The effective address of the page to invalidate.
15338 *
15339 * @remarks In ring-0 not all of the state needs to be synced in.
15340 */
15341VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
15342{
15343 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15344
15345 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15346 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
15347 Assert(!pVCpu->iem.s.cActiveMappings);
15348 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15349}
15350
15351
15352/**
15353 * Interface for HM and EM to emulate the INVPCID instruction.
15354 *
15355 * @returns Strict VBox status code.
15356 * @retval VINF_PGM_SYNC_CR3
15357 *
15358 * @param pVCpu The cross context virtual CPU structure.
15359 * @param cbInstr The instruction length in bytes.
15360 * @param iEffSeg The effective segment register.
15361 * @param GCPtrDesc The effective address of the INVPCID descriptor.
15362 * @param uType The invalidation type.
15363 *
15364 * @remarks In ring-0 not all of the state needs to be synced in.
15365 */
15366VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
15367 uint64_t uType)
15368{
15369 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
15370
15371 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15372 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
15373 Assert(!pVCpu->iem.s.cActiveMappings);
15374 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15375}
15376
15377
15378/**
15379 * Interface for HM and EM to emulate the CPUID instruction.
15380 *
15381 * @returns Strict VBox status code.
15382 *
15383 * @param pVCpu The cross context virtual CPU structure.
15384 * @param cbInstr The instruction length in bytes.
15385 *
15386 * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
15387 */
15388VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
15389{
15390 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15391 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
15392
15393 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15394 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
15395 Assert(!pVCpu->iem.s.cActiveMappings);
15396 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15397}
15398
15399
15400/**
15401 * Interface for HM and EM to emulate the RDPMC instruction.
15402 *
15403 * @returns Strict VBox status code.
15404 *
15405 * @param pVCpu The cross context virtual CPU structure.
15406 * @param cbInstr The instruction length in bytes.
15407 *
15408 * @remarks Not all of the state needs to be synced in.
15409 */
15410VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
15411{
15412 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15413 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
15414
15415 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15416 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
15417 Assert(!pVCpu->iem.s.cActiveMappings);
15418 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15419}
15420
15421
15422/**
15423 * Interface for HM and EM to emulate the RDTSC instruction.
15424 *
15425 * @returns Strict VBox status code.
15426 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15427 *
15428 * @param pVCpu The cross context virtual CPU structure.
15429 * @param cbInstr The instruction length in bytes.
15430 *
15431 * @remarks Not all of the state needs to be synced in.
15432 */
15433VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
15434{
15435 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15436 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
15437
15438 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15439 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
15440 Assert(!pVCpu->iem.s.cActiveMappings);
15441 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15442}
15443
15444
15445/**
15446 * Interface for HM and EM to emulate the RDTSCP instruction.
15447 *
15448 * @returns Strict VBox status code.
15449 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15450 *
15451 * @param pVCpu The cross context virtual CPU structure.
15452 * @param cbInstr The instruction length in bytes.
15453 *
15454 * @remarks Not all of the state needs to be synced in. Recommended
15455 * to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
15456 */
15457VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
15458{
15459 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15460 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
15461
15462 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15463 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
15464 Assert(!pVCpu->iem.s.cActiveMappings);
15465 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15466}
15467
15468
15469/**
15470 * Interface for HM and EM to emulate the RDMSR instruction.
15471 *
15472 * @returns Strict VBox status code.
15473 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15474 *
15475 * @param pVCpu The cross context virtual CPU structure.
15476 * @param cbInstr The instruction length in bytes.
15477 *
15478 * @remarks Not all of the state needs to be synced in. Requires RCX and
15479 * (currently) all MSRs.
15480 */
15481VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
15482{
15483 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15484 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS);
15485
15486 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15487 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
15488 Assert(!pVCpu->iem.s.cActiveMappings);
15489 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15490}
15491
15492
15493/**
15494 * Interface for HM and EM to emulate the WRMSR instruction.
15495 *
15496 * @returns Strict VBox status code.
15497 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15498 *
15499 * @param pVCpu The cross context virtual CPU structure.
15500 * @param cbInstr The instruction length in bytes.
15501 *
15502 * @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
15503 * and (currently) all MSRs.
15504 */
15505VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
15506{
15507 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15508 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
15509 | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS);
15510
15511 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15512 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
15513 Assert(!pVCpu->iem.s.cActiveMappings);
15514 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15515}
15516
15517
15518/**
15519 * Interface for HM and EM to emulate the MONITOR instruction.
15520 *
15521 * @returns Strict VBox status code.
15522 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15523 *
15524 * @param pVCpu The cross context virtual CPU structure.
15525 * @param cbInstr The instruction length in bytes.
15526 *
15527 * @remarks Not all of the state needs to be synced in.
15528 * @remarks ASSUMES the default segment of DS and no segment override prefixes
15529 * are used.
15530 */
15531VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
15532{
15533 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15534 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
15535
15536 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15537 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
15538 Assert(!pVCpu->iem.s.cActiveMappings);
15539 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15540}
15541
15542
15543/**
15544 * Interface for HM and EM to emulate the MWAIT instruction.
15545 *
15546 * @returns Strict VBox status code.
15547 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15548 *
15549 * @param pVCpu The cross context virtual CPU structure.
15550 * @param cbInstr The instruction length in bytes.
15551 *
15552 * @remarks Not all of the state needs to be synced in.
15553 */
15554VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
15555{
15556 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15557 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX);
15558
15559 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15560 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
15561 Assert(!pVCpu->iem.s.cActiveMappings);
15562 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15563}
15564
15565
15566/**
15567 * Interface for HM and EM to emulate the HLT instruction.
15568 *
15569 * @returns Strict VBox status code.
15570 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15571 *
15572 * @param pVCpu The cross context virtual CPU structure.
15573 * @param cbInstr The instruction length in bytes.
15574 *
15575 * @remarks Not all of the state needs to be synced in.
15576 */
15577VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
15578{
15579 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15580
15581 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15582 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
15583 Assert(!pVCpu->iem.s.cActiveMappings);
15584 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15585}
15586
15587
15588/**
15589 * Checks if IEM is in the process of delivering an event (interrupt or
15590 * exception).
15591 *
15592 * @returns true if we're in the process of raising an interrupt or exception,
15593 * false otherwise.
15594 * @param pVCpu The cross context virtual CPU structure.
15595 * @param puVector Where to store the vector associated with the
15596 * currently delivered event, optional.
15597 * @param pfFlags Where to store th event delivery flags (see
15598 * IEM_XCPT_FLAGS_XXX), optional.
15599 * @param puErr Where to store the error code associated with the
15600 * event, optional.
15601 * @param puCr2 Where to store the CR2 associated with the event,
15602 * optional.
15603 * @remarks The caller should check the flags to determine if the error code and
15604 * CR2 are valid for the event.
15605 */
15606VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2)
15607{
15608 bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
15609 if (fRaisingXcpt)
15610 {
15611 if (puVector)
15612 *puVector = pVCpu->iem.s.uCurXcpt;
15613 if (pfFlags)
15614 *pfFlags = pVCpu->iem.s.fCurXcpt;
15615 if (puErr)
15616 *puErr = pVCpu->iem.s.uCurXcptErr;
15617 if (puCr2)
15618 *puCr2 = pVCpu->iem.s.uCurXcptCr2;
15619 }
15620 return fRaisingXcpt;
15621}
15622
15623#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
15624
15625/**
15626 * Interface for HM and EM to emulate the CLGI instruction.
15627 *
15628 * @returns Strict VBox status code.
15629 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15630 * @param cbInstr The instruction length in bytes.
15631 * @thread EMT(pVCpu)
15632 */
15633VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClgi(PVMCPUCC pVCpu, uint8_t cbInstr)
15634{
15635 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15636
15637 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15638 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clgi);
15639 Assert(!pVCpu->iem.s.cActiveMappings);
15640 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15641}
15642
15643
15644/**
15645 * Interface for HM and EM to emulate the STGI instruction.
15646 *
15647 * @returns Strict VBox status code.
15648 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15649 * @param cbInstr The instruction length in bytes.
15650 * @thread EMT(pVCpu)
15651 */
15652VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedStgi(PVMCPUCC pVCpu, uint8_t cbInstr)
15653{
15654 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15655
15656 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15657 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_stgi);
15658 Assert(!pVCpu->iem.s.cActiveMappings);
15659 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15660}
15661
15662
15663/**
15664 * Interface for HM and EM to emulate the VMLOAD instruction.
15665 *
15666 * @returns Strict VBox status code.
15667 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15668 * @param cbInstr The instruction length in bytes.
15669 * @thread EMT(pVCpu)
15670 */
15671VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmload(PVMCPUCC pVCpu, uint8_t cbInstr)
15672{
15673 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15674
15675 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15676 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmload);
15677 Assert(!pVCpu->iem.s.cActiveMappings);
15678 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15679}
15680
15681
15682/**
15683 * Interface for HM and EM to emulate the VMSAVE instruction.
15684 *
15685 * @returns Strict VBox status code.
15686 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15687 * @param cbInstr The instruction length in bytes.
15688 * @thread EMT(pVCpu)
15689 */
15690VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmsave(PVMCPUCC pVCpu, uint8_t cbInstr)
15691{
15692 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15693
15694 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15695 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmsave);
15696 Assert(!pVCpu->iem.s.cActiveMappings);
15697 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15698}
15699
15700
15701/**
15702 * Interface for HM and EM to emulate the INVLPGA instruction.
15703 *
15704 * @returns Strict VBox status code.
15705 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15706 * @param cbInstr The instruction length in bytes.
15707 * @thread EMT(pVCpu)
15708 */
15709VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpga(PVMCPUCC pVCpu, uint8_t cbInstr)
15710{
15711 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15712
15713 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15714 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invlpga);
15715 Assert(!pVCpu->iem.s.cActiveMappings);
15716 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15717}
15718
15719
15720/**
15721 * Interface for HM and EM to emulate the VMRUN instruction.
15722 *
15723 * @returns Strict VBox status code.
15724 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15725 * @param cbInstr The instruction length in bytes.
15726 * @thread EMT(pVCpu)
15727 */
15728VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmrun(PVMCPUCC pVCpu, uint8_t cbInstr)
15729{
15730 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15731 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK);
15732
15733 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15734 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmrun);
15735 Assert(!pVCpu->iem.s.cActiveMappings);
15736 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15737}
15738
15739
15740/**
15741 * Interface for HM and EM to emulate \#VMEXIT.
15742 *
15743 * @returns Strict VBox status code.
15744 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15745 * @param uExitCode The exit code.
15746 * @param uExitInfo1 The exit info. 1 field.
15747 * @param uExitInfo2 The exit info. 2 field.
15748 * @thread EMT(pVCpu)
15749 */
15750VMM_INT_DECL(VBOXSTRICTRC) IEMExecSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2)
15751{
15752 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
15753 VBOXSTRICTRC rcStrict = iemSvmVmexit(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
15754 if (pVCpu->iem.s.cActiveMappings)
15755 iemMemRollback(pVCpu);
15756 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15757}
15758
15759#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
15760
15761#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15762
15763/**
15764 * Interface for HM and EM to read a VMCS field from the nested-guest VMCS.
15765 *
15766 * It is ASSUMED the caller knows what they're doing. No VMREAD instruction checks
15767 * are performed. Bounds checks are strict builds only.
15768 *
15769 * @param pVmcs Pointer to the virtual VMCS.
15770 * @param u64VmcsField The VMCS field.
15771 * @param pu64Dst Where to store the VMCS value.
15772 *
15773 * @remarks May be called with interrupts disabled.
15774 * @todo This should probably be moved to CPUM someday.
15775 */
15776VMM_INT_DECL(void) IEMReadVmxVmcsField(PCVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t *pu64Dst)
15777{
15778 AssertPtr(pVmcs);
15779 AssertPtr(pu64Dst);
15780 iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField);
15781}
15782
15783
15784/**
15785 * Interface for HM and EM to write a VMCS field in the nested-guest VMCS.
15786 *
15787 * It is ASSUMED the caller knows what they're doing. No VMWRITE instruction checks
15788 * are performed. Bounds checks are strict builds only.
15789 *
15790 * @param pVmcs Pointer to the virtual VMCS.
15791 * @param u64VmcsField The VMCS field.
15792 * @param u64Val The value to write.
15793 *
15794 * @remarks May be called with interrupts disabled.
15795 * @todo This should probably be moved to CPUM someday.
15796 */
15797VMM_INT_DECL(void) IEMWriteVmxVmcsField(PVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t u64Val)
15798{
15799 AssertPtr(pVmcs);
15800 iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField);
15801}
15802
15803
15804/**
15805 * Interface for HM and EM to virtualize x2APIC MSR accesses.
15806 *
15807 * @returns Strict VBox status code.
15808 * @retval VINF_VMX_MODIFIES_BEHAVIOR if the MSR access was virtualized.
15809 * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR access must be handled by
15810 * the x2APIC device.
15811 * @retval VERR_OUT_RANGE if the caller must raise \#GP(0).
15812 *
15813 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15814 * @param idMsr The MSR being read.
15815 * @param pu64Value Pointer to the value being written or where to store the
15816 * value being read.
15817 * @param fWrite Whether this is an MSR write or read access.
15818 * @thread EMT(pVCpu)
15819 */
15820VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVirtApicAccessMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value, bool fWrite)
15821{
15822 Assert(pu64Value);
15823
15824 VBOXSTRICTRC rcStrict;
15825 if (fWrite)
15826 rcStrict = iemVmxVirtApicAccessMsrWrite(pVCpu, idMsr, *pu64Value);
15827 else
15828 rcStrict = iemVmxVirtApicAccessMsrRead(pVCpu, idMsr, pu64Value);
15829 Assert(!pVCpu->iem.s.cActiveMappings);
15830 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15831
15832}
15833
15834
15835/**
15836 * Interface for HM and EM to virtualize memory-mapped APIC accesses.
15837 *
15838 * @returns Strict VBox status code.
15839 * @retval VINF_VMX_MODIFIES_BEHAVIOR if the memory access was virtualized.
15840 * @retval VINF_VMX_VMEXIT if the access causes a VM-exit.
15841 *
15842 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15843 * @param pExitInfo Pointer to the VM-exit information.
15844 * @param pExitEventInfo Pointer to the VM-exit event information.
15845 * @thread EMT(pVCpu)
15846 */
15847VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicAccess(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
15848{
15849 Assert(pExitInfo);
15850 Assert(pExitEventInfo);
15851 VBOXSTRICTRC rcStrict = iemVmxVmexitApicAccessWithInfo(pVCpu, pExitInfo, pExitEventInfo);
15852 Assert(!pVCpu->iem.s.cActiveMappings);
15853 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15854
15855}
15856
15857
15858/**
15859 * Interface for HM and EM to perform an APIC-write emulation which may cause a
15860 * VM-exit.
15861 *
15862 * @returns Strict VBox status code.
15863 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15864 * @thread EMT(pVCpu)
15865 */
15866VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicWrite(PVMCPUCC pVCpu)
15867{
15868 VBOXSTRICTRC rcStrict = iemVmxApicWriteEmulation(pVCpu);
15869 Assert(!pVCpu->iem.s.cActiveMappings);
15870 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15871}
15872
15873
15874/**
15875 * Interface for HM and EM to emulate VM-exit due to expiry of the preemption timer.
15876 *
15877 * @returns Strict VBox status code.
15878 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15879 * @thread EMT(pVCpu)
15880 */
15881VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitPreemptTimer(PVMCPUCC pVCpu)
15882{
15883 VBOXSTRICTRC rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
15884 Assert(!pVCpu->iem.s.cActiveMappings);
15885 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15886}
15887
15888
15889/**
15890 * Interface for HM and EM to emulate VM-exit due to external interrupts.
15891 *
15892 * @returns Strict VBox status code.
15893 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15894 * @param uVector The external interrupt vector (pass 0 if the external
15895 * interrupt is still pending).
15896 * @param fIntPending Whether the external interrupt is pending or
15897 * acknowdledged in the interrupt controller.
15898 * @thread EMT(pVCpu)
15899 */
15900VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending)
15901{
15902 VBOXSTRICTRC rcStrict = iemVmxVmexitExtInt(pVCpu, uVector, fIntPending);
15903 Assert(!pVCpu->iem.s.cActiveMappings);
15904 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15905}
15906
15907
15908/**
15909 * Interface for HM and EM to emulate VM-exit due to exceptions.
15910 *
15911 * Exception includes NMIs, software exceptions (those generated by INT3 or
15912 * INTO) and privileged software exceptions (those generated by INT1/ICEBP).
15913 *
15914 * @returns Strict VBox status code.
15915 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15916 * @param pExitInfo Pointer to the VM-exit information.
15917 * @param pExitEventInfo Pointer to the VM-exit event information.
15918 * @thread EMT(pVCpu)
15919 */
15920VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcpt(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
15921{
15922 Assert(pExitInfo);
15923 Assert(pExitEventInfo);
15924 VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, pExitInfo, pExitEventInfo);
15925 Assert(!pVCpu->iem.s.cActiveMappings);
15926 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15927}
15928
15929
15930/**
15931 * Interface for HM and EM to emulate VM-exit due to NMIs.
15932 *
15933 * @returns Strict VBox status code.
15934 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15935 * @thread EMT(pVCpu)
15936 */
15937VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcptNmi(PVMCPUCC pVCpu)
15938{
15939 VMXVEXITINFO ExitInfo;
15940 RT_ZERO(ExitInfo);
15941 ExitInfo.uReason = VMX_EXIT_XCPT_OR_NMI;
15942
15943 VMXVEXITEVENTINFO ExitEventInfo;
15944 RT_ZERO(ExitEventInfo);
15945 ExitEventInfo.uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1)
15946 | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_NMI)
15947 | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, X86_XCPT_NMI);
15948
15949 VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, &ExitInfo, &ExitEventInfo);
15950 Assert(!pVCpu->iem.s.cActiveMappings);
15951 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15952}
15953
15954
15955/**
15956 * Interface for HM and EM to emulate VM-exit due to a triple-fault.
15957 *
15958 * @returns Strict VBox status code.
15959 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15960 * @thread EMT(pVCpu)
15961 */
15962VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTripleFault(PVMCPUCC pVCpu)
15963{
15964 VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
15965 Assert(!pVCpu->iem.s.cActiveMappings);
15966 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15967}
15968
15969
15970/**
15971 * Interface for HM and EM to emulate VM-exit due to startup-IPI (SIPI).
15972 *
15973 * @returns Strict VBox status code.
15974 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15975 * @param uVector The SIPI vector.
15976 * @thread EMT(pVCpu)
15977 */
15978VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitStartupIpi(PVMCPUCC pVCpu, uint8_t uVector)
15979{
15980 VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_SIPI, uVector);
15981 Assert(!pVCpu->iem.s.cActiveMappings);
15982 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15983}
15984
15985
15986/**
15987 * Interface for HM and EM to emulate a VM-exit.
15988 *
15989 * If a specialized version of a VM-exit handler exists, that must be used instead.
15990 *
15991 * @returns Strict VBox status code.
15992 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15993 * @param uExitReason The VM-exit reason.
15994 * @param u64ExitQual The Exit qualification.
15995 * @thread EMT(pVCpu)
15996 */
15997VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual)
15998{
15999 VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, uExitReason, u64ExitQual);
16000 Assert(!pVCpu->iem.s.cActiveMappings);
16001 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
16002}
16003
16004
16005/**
16006 * Interface for HM and EM to emulate a VM-exit due to an instruction.
16007 *
16008 * This is meant to be used for those instructions that VMX provides additional
16009 * decoding information beyond just the instruction length!
16010 *
16011 * @returns Strict VBox status code.
16012 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16013 * @param pExitInfo Pointer to the VM-exit information.
16014 * @thread EMT(pVCpu)
16015 */
16016VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16017{
16018 VBOXSTRICTRC rcStrict = iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
16019 Assert(!pVCpu->iem.s.cActiveMappings);
16020 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
16021}
16022
16023
16024/**
16025 * Interface for HM and EM to emulate a VM-exit due to an instruction.
16026 *
16027 * This is meant to be used for those instructions that VMX provides only the
16028 * instruction length.
16029 *
16030 * @returns Strict VBox status code.
16031 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16032 * @param pExitInfo Pointer to the VM-exit information.
16033 * @param cbInstr The instruction length in bytes.
16034 * @thread EMT(pVCpu)
16035 */
16036VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr)
16037{
16038 VBOXSTRICTRC rcStrict = iemVmxVmexitInstr(pVCpu, uExitReason, cbInstr);
16039 Assert(!pVCpu->iem.s.cActiveMappings);
16040 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
16041}
16042
16043
16044/**
16045 * Interface for HM and EM to emulate a trap-like VM-exit (MTF, APIC-write,
16046 * Virtualized-EOI, TPR-below threshold).
16047 *
16048 * @returns Strict VBox status code.
16049 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16050 * @param pExitInfo Pointer to the VM-exit information.
16051 * @thread EMT(pVCpu)
16052 */
16053VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTrapLike(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16054{
16055 Assert(pExitInfo);
16056 VBOXSTRICTRC rcStrict = iemVmxVmexitTrapLikeWithInfo(pVCpu, pExitInfo);
16057 Assert(!pVCpu->iem.s.cActiveMappings);
16058 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
16059}
16060
16061
16062/**
16063 * Interface for HM and EM to emulate a VM-exit due to a task switch.
16064 *
16065 * @returns Strict VBox status code.
16066 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16067 * @param pExitInfo Pointer to the VM-exit information.
16068 * @param pExitEventInfo Pointer to the VM-exit event information.
16069 * @thread EMT(pVCpu)
16070 */
16071VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTaskSwitch(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
16072{
16073 Assert(pExitInfo);
16074 Assert(pExitEventInfo);
16075 Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH);
16076 VBOXSTRICTRC rcStrict = iemVmxVmexitTaskSwitchWithInfo(pVCpu, pExitInfo, pExitEventInfo);
16077 Assert(!pVCpu->iem.s.cActiveMappings);
16078 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
16079}
16080
16081
16082/**
16083 * Interface for HM and EM to emulate the VMREAD instruction.
16084 *
16085 * @returns Strict VBox status code.
16086 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16087 * @param pExitInfo Pointer to the VM-exit information.
16088 * @thread EMT(pVCpu)
16089 */
16090VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmread(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16091{
16092 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16093 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16094 Assert(pExitInfo);
16095
16096 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16097
16098 VBOXSTRICTRC rcStrict;
16099 uint8_t const cbInstr = pExitInfo->cbInstr;
16100 bool const fIs64BitMode = RT_BOOL(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
16101 uint64_t const u64FieldEnc = fIs64BitMode
16102 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
16103 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
16104 if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
16105 {
16106 if (fIs64BitMode)
16107 {
16108 uint64_t *pu64Dst = iemGRegRefU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
16109 rcStrict = iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64FieldEnc, pExitInfo);
16110 }
16111 else
16112 {
16113 uint32_t *pu32Dst = iemGRegRefU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
16114 rcStrict = iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u64FieldEnc, pExitInfo);
16115 }
16116 }
16117 else
16118 {
16119 RTGCPTR const GCPtrDst = pExitInfo->GCPtrEffAddr;
16120 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
16121 rcStrict = iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64FieldEnc, pExitInfo);
16122 }
16123 Assert(!pVCpu->iem.s.cActiveMappings);
16124 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16125}
16126
16127
16128/**
16129 * Interface for HM and EM to emulate the VMWRITE instruction.
16130 *
16131 * @returns Strict VBox status code.
16132 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16133 * @param pExitInfo Pointer to the VM-exit information.
16134 * @thread EMT(pVCpu)
16135 */
16136VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmwrite(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16137{
16138 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16139 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16140 Assert(pExitInfo);
16141
16142 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16143
16144 uint64_t u64Val;
16145 uint8_t iEffSeg;
16146 if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
16147 {
16148 u64Val = iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
16149 iEffSeg = UINT8_MAX;
16150 }
16151 else
16152 {
16153 u64Val = pExitInfo->GCPtrEffAddr;
16154 iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
16155 }
16156 uint8_t const cbInstr = pExitInfo->cbInstr;
16157 uint64_t const u64FieldEnc = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
16158 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
16159 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
16160 VBOXSTRICTRC rcStrict = iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, u64Val, u64FieldEnc, pExitInfo);
16161 Assert(!pVCpu->iem.s.cActiveMappings);
16162 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16163}
16164
16165
16166/**
16167 * Interface for HM and EM to emulate the VMPTRLD instruction.
16168 *
16169 * @returns Strict VBox status code.
16170 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16171 * @param pExitInfo Pointer to the VM-exit information.
16172 * @thread EMT(pVCpu)
16173 */
16174VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrld(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16175{
16176 Assert(pExitInfo);
16177 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16178 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16179
16180 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16181
16182 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16183 uint8_t const cbInstr = pExitInfo->cbInstr;
16184 RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
16185 VBOXSTRICTRC rcStrict = iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
16186 Assert(!pVCpu->iem.s.cActiveMappings);
16187 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16188}
16189
16190
16191/**
16192 * Interface for HM and EM to emulate the VMPTRST instruction.
16193 *
16194 * @returns Strict VBox status code.
16195 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16196 * @param pExitInfo Pointer to the VM-exit information.
16197 * @thread EMT(pVCpu)
16198 */
16199VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrst(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16200{
16201 Assert(pExitInfo);
16202 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16203 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16204
16205 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16206
16207 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16208 uint8_t const cbInstr = pExitInfo->cbInstr;
16209 RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
16210 VBOXSTRICTRC rcStrict = iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
16211 Assert(!pVCpu->iem.s.cActiveMappings);
16212 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16213}
16214
16215
16216/**
16217 * Interface for HM and EM to emulate the VMCLEAR instruction.
16218 *
16219 * @returns Strict VBox status code.
16220 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16221 * @param pExitInfo Pointer to the VM-exit information.
16222 * @thread EMT(pVCpu)
16223 */
16224VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmclear(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16225{
16226 Assert(pExitInfo);
16227 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16228 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16229
16230 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16231
16232 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16233 uint8_t const cbInstr = pExitInfo->cbInstr;
16234 RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
16235 VBOXSTRICTRC rcStrict = iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
16236 Assert(!pVCpu->iem.s.cActiveMappings);
16237 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16238}
16239
16240
16241/**
16242 * Interface for HM and EM to emulate the VMLAUNCH/VMRESUME instruction.
16243 *
16244 * @returns Strict VBox status code.
16245 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16246 * @param cbInstr The instruction length in bytes.
16247 * @param uInstrId The instruction ID (VMXINSTRID_VMLAUNCH or
16248 * VMXINSTRID_VMRESUME).
16249 * @thread EMT(pVCpu)
16250 */
16251VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId)
16252{
16253 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
16254 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK);
16255
16256 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16257 VBOXSTRICTRC rcStrict = iemVmxVmlaunchVmresume(pVCpu, cbInstr, uInstrId);
16258 Assert(!pVCpu->iem.s.cActiveMappings);
16259 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16260}
16261
16262
16263/**
16264 * Interface for HM and EM to emulate the VMXON instruction.
16265 *
16266 * @returns Strict VBox status code.
16267 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16268 * @param pExitInfo Pointer to the VM-exit information.
16269 * @thread EMT(pVCpu)
16270 */
16271VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxon(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16272{
16273 Assert(pExitInfo);
16274 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
16275 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16276
16277 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16278
16279 uint8_t const iEffSeg = pExitInfo->InstrInfo.VmxXsave.iSegReg;
16280 uint8_t const cbInstr = pExitInfo->cbInstr;
16281 RTGCPTR const GCPtrVmxon = pExitInfo->GCPtrEffAddr;
16282 VBOXSTRICTRC rcStrict = iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, pExitInfo);
16283 Assert(!pVCpu->iem.s.cActiveMappings);
16284 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16285}
16286
16287
16288/**
16289 * Interface for HM and EM to emulate the VMXOFF instruction.
16290 *
16291 * @returns Strict VBox status code.
16292 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16293 * @param cbInstr The instruction length in bytes.
16294 * @thread EMT(pVCpu)
16295 */
16296VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxoff(PVMCPUCC pVCpu, uint8_t cbInstr)
16297{
16298 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
16299 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16300
16301 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16302 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmxoff);
16303 Assert(!pVCpu->iem.s.cActiveMappings);
16304 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16305}
16306
16307
16308/**
16309 * Interface for HM and EM to emulate the INVVPID instruction.
16310 *
16311 * @returns Strict VBox status code.
16312 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16313 * @param pExitInfo Pointer to the VM-exit information.
16314 * @thread EMT(pVCpu)
16315 */
16316VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvvpid(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16317{
16318 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4);
16319 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16320 Assert(pExitInfo);
16321
16322 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16323
16324 uint8_t const iEffSeg = pExitInfo->InstrInfo.Inv.iSegReg;
16325 uint8_t const cbInstr = pExitInfo->cbInstr;
16326 RTGCPTR const GCPtrInvvpidDesc = pExitInfo->GCPtrEffAddr;
16327 uint64_t const u64InvvpidType = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
16328 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2)
16329 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2);
16330 VBOXSTRICTRC rcStrict = iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, u64InvvpidType, pExitInfo);
16331 Assert(!pVCpu->iem.s.cActiveMappings);
16332 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16333}
16334
16335
16336# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
16337/**
16338 * Interface for HM and EM to emulate the INVEPT instruction.
16339 *
16340 * @returns Strict VBox status code.
16341 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16342 * @param pExitInfo Pointer to the VM-exit information.
16343 * @thread EMT(pVCpu)
16344 */
16345VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvept(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
16346{
16347 IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4);
16348 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
16349 Assert(pExitInfo);
16350
16351 iemInitExec(pVCpu, false /*fBypassHandlers*/);
16352
16353 uint8_t const iEffSeg = pExitInfo->InstrInfo.Inv.iSegReg;
16354 uint8_t const cbInstr = pExitInfo->cbInstr;
16355 RTGCPTR const GCPtrInveptDesc = pExitInfo->GCPtrEffAddr;
16356 uint64_t const u64InveptType = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
16357 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2)
16358 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2);
16359 VBOXSTRICTRC rcStrict = iemVmxInvept(pVCpu, cbInstr, iEffSeg, GCPtrInveptDesc, u64InveptType, pExitInfo);
16360 Assert(!pVCpu->iem.s.cActiveMappings);
16361 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
16362}
16363# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
16364
16365
16366/**
16367 * @callback_method_impl{FNPGMPHYSHANDLER, VMX APIC-access page accesses}
16368 *
16369 * @remarks The @a pvUser argument is currently unused.
16370 */
16371PGM_ALL_CB2_DECL(VBOXSTRICTRC) iemVmxApicAccessPageHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys,
16372 void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType,
16373 PGMACCESSORIGIN enmOrigin, void *pvUser)
16374{
16375 RT_NOREF3(pvPhys, enmOrigin, pvUser);
16376
16377 RTGCPHYS const GCPhysAccessBase = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
16378 if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
16379 {
16380 Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
16381 Assert(CPUMGetGuestVmxApicAccessPageAddr(IEM_GET_CTX(pVCpu)) == GCPhysAccessBase);
16382
16383 /** @todo NSTVMX: How are we to distinguish instruction fetch accesses here?
16384 * Currently they will go through as read accesses. */
16385 uint32_t const fAccess = enmAccessType == PGMACCESSTYPE_WRITE ? IEM_ACCESS_TYPE_WRITE : IEM_ACCESS_TYPE_READ;
16386 uint16_t const offAccess = GCPhysFault & GUEST_PAGE_OFFSET_MASK;
16387 VBOXSTRICTRC rcStrict = iemVmxVirtApicAccessMem(pVCpu, offAccess, cbBuf, pvBuf, fAccess);
16388 if (RT_FAILURE(rcStrict))
16389 return rcStrict;
16390
16391 /* Any access on this APIC-access page has been handled, caller should not carry out the access. */
16392 return VINF_SUCCESS;
16393 }
16394
16395 Log(("iemVmxApicAccessPageHandler: Access outside VMX non-root mode, deregistering page at %#RGp\n", GCPhysAccessBase));
16396 int rc = PGMHandlerPhysicalDeregister(pVM, GCPhysAccessBase);
16397 if (RT_FAILURE(rc))
16398 return rc;
16399
16400 /* Instruct the caller of this handler to perform the read/write as normal memory. */
16401 return VINF_PGM_HANDLER_DO_DEFAULT;
16402}
16403
16404#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
16405
16406#ifdef IN_RING3
16407
16408/**
16409 * Handles the unlikely and probably fatal merge cases.
16410 *
16411 * @returns Merged status code.
16412 * @param rcStrict Current EM status code.
16413 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
16414 * with @a rcStrict.
16415 * @param iMemMap The memory mapping index. For error reporting only.
16416 * @param pVCpu The cross context virtual CPU structure of the calling
16417 * thread, for error reporting only.
16418 */
16419DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
16420 unsigned iMemMap, PVMCPUCC pVCpu)
16421{
16422 if (RT_FAILURE_NP(rcStrict))
16423 return rcStrict;
16424
16425 if (RT_FAILURE_NP(rcStrictCommit))
16426 return rcStrictCommit;
16427
16428 if (rcStrict == rcStrictCommit)
16429 return rcStrictCommit;
16430
16431 AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
16432 VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
16433 pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
16434 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
16435 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
16436 return VERR_IOM_FF_STATUS_IPE;
16437}
16438
16439
16440/**
16441 * Helper for IOMR3ProcessForceFlag.
16442 *
16443 * @returns Merged status code.
16444 * @param rcStrict Current EM status code.
16445 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
16446 * with @a rcStrict.
16447 * @param iMemMap The memory mapping index. For error reporting only.
16448 * @param pVCpu The cross context virtual CPU structure of the calling
16449 * thread, for error reporting only.
16450 */
16451DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
16452{
16453 /* Simple. */
16454 if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))
16455 return rcStrictCommit;
16456
16457 if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
16458 return rcStrict;
16459
16460 /* EM scheduling status codes. */
16461 if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
16462 && rcStrict <= VINF_EM_LAST))
16463 {
16464 if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
16465 && rcStrictCommit <= VINF_EM_LAST))
16466 return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
16467 }
16468
16469 /* Unlikely */
16470 return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
16471}
16472
16473
16474/**
16475 * Called by force-flag handling code when VMCPU_FF_IEM is set.
16476 *
16477 * @returns Merge between @a rcStrict and what the commit operation returned.
16478 * @param pVM The cross context VM structure.
16479 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
16480 * @param rcStrict The status code returned by ring-0 or raw-mode.
16481 */
16482VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
16483{
16484 /*
16485 * Reset the pending commit.
16486 */
16487 AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)
16488 & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),
16489 ("%#x %#x %#x\n",
16490 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
16491 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
16492
16493 /*
16494 * Commit the pending bounce buffers (usually just one).
16495 */
16496 unsigned cBufs = 0;
16497 unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
16498 while (iMemMap-- > 0)
16499 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))
16500 {
16501 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
16502 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
16503 Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
16504
16505 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
16506 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
16507 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
16508
16509 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
16510 {
16511 VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
16512 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
16513 pbBuf,
16514 cbFirst,
16515 PGMACCESSORIGIN_IEM);
16516 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
16517 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
16518 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
16519 VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
16520 }
16521
16522 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
16523 {
16524 VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
16525 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
16526 pbBuf + cbFirst,
16527 cbSecond,
16528 PGMACCESSORIGIN_IEM);
16529 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
16530 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
16531 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
16532 VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
16533 }
16534 cBufs++;
16535 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
16536 }
16537
16538 AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
16539 ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
16540 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
16541 pVCpu->iem.s.cActiveMappings = 0;
16542 return rcStrict;
16543}
16544
16545#endif /* IN_RING3 */
16546
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette