VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAll.cpp@ 73617

Last change on this file since 73617 was 73608, checked in by vboxsync, 7 years ago

VMM/IEM: Nested VMX: bugref:9180 Doxygen fix.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 620.9 KB
Line 
1/* $Id: IEMAll.cpp 73608 2018-08-10 07:49:12Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - All Contexts.
4 */
5
6/*
7 * Copyright (C) 2011-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/** @page pg_iem IEM - Interpreted Execution Manager
20 *
21 * The interpreted exeuction manager (IEM) is for executing short guest code
22 * sequences that are causing too many exits / virtualization traps. It will
23 * also be used to interpret single instructions, thus replacing the selective
24 * interpreters in EM and IOM.
25 *
26 * Design goals:
27 * - Relatively small footprint, although we favour speed and correctness
28 * over size.
29 * - Reasonably fast.
30 * - Correctly handle lock prefixed instructions.
31 * - Complete instruction set - eventually.
32 * - Refactorable into a recompiler, maybe.
33 * - Replace EMInterpret*.
34 *
35 * Using the existing disassembler has been considered, however this is thought
36 * to conflict with speed as the disassembler chews things a bit too much while
37 * leaving us with a somewhat complicated state to interpret afterwards.
38 *
39 *
40 * The current code is very much work in progress. You've been warned!
41 *
42 *
43 * @section sec_iem_fpu_instr FPU Instructions
44 *
45 * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
46 * same or equivalent instructions on the host FPU. To make life easy, we also
47 * let the FPU prioritize the unmasked exceptions for us. This however, only
48 * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
49 * for FPU exception delivery, because with CR0.NE=0 there is a window where we
50 * can trigger spurious FPU exceptions.
51 *
52 * The guest FPU state is not loaded into the host CPU and kept there till we
53 * leave IEM because the calling conventions have declared an all year open
54 * season on much of the FPU state. For instance an innocent looking call to
55 * memcpy might end up using a whole bunch of XMM or MM registers if the
56 * particular implementation finds it worthwhile.
57 *
58 *
59 * @section sec_iem_logging Logging
60 *
61 * The IEM code uses the \"IEM\" log group for the main logging. The different
62 * logging levels/flags are generally used for the following purposes:
63 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events.
64 * - Flow (LogFlow): Basic enter/exit IEM state info.
65 * - Level 2 (Log2): ?
66 * - Level 3 (Log3): More detailed enter/exit IEM state info.
67 * - Level 4 (Log4): Decoding mnemonics w/ EIP.
68 * - Level 5 (Log5): Decoding details.
69 * - Level 6 (Log6): Enables/disables the lockstep comparison with REM.
70 * - Level 7 (Log7): iret++ execution logging.
71 * - Level 8 (Log8): Memory writes.
72 * - Level 9 (Log9): Memory reads.
73 *
74 */
75
76//#define IEM_LOG_MEMORY_WRITES
77#define IEM_IMPLEMENTS_TASKSWITCH
78
79/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
80#ifdef _MSC_VER
81# pragma warning(disable:4505)
82#endif
83
84
85/*********************************************************************************************************************************
86* Header Files *
87*********************************************************************************************************************************/
88#define LOG_GROUP LOG_GROUP_IEM
89#define VMCPU_INCL_CPUM_GST_CTX
90#include <VBox/vmm/iem.h>
91#include <VBox/vmm/cpum.h>
92#include <VBox/vmm/apic.h>
93#include <VBox/vmm/pdm.h>
94#include <VBox/vmm/pgm.h>
95#include <VBox/vmm/iom.h>
96#include <VBox/vmm/em.h>
97#include <VBox/vmm/hm.h>
98#include <VBox/vmm/nem.h>
99#include <VBox/vmm/gim.h>
100#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
101# include <VBox/vmm/em.h>
102# include <VBox/vmm/hm_svm.h>
103#endif
104#include <VBox/vmm/tm.h>
105#include <VBox/vmm/dbgf.h>
106#include <VBox/vmm/dbgftrace.h>
107#ifdef VBOX_WITH_RAW_MODE_NOT_R0
108# include <VBox/vmm/patm.h>
109# if defined(VBOX_WITH_CALL_RECORD) || defined(REM_MONITOR_CODE_PAGES)
110# include <VBox/vmm/csam.h>
111# endif
112#endif
113#include "IEMInternal.h"
114#include <VBox/vmm/vm.h>
115#include <VBox/log.h>
116#include <VBox/err.h>
117#include <VBox/param.h>
118#include <VBox/dis.h>
119#include <VBox/disopcode.h>
120#include <iprt/assert.h>
121#include <iprt/string.h>
122#include <iprt/x86.h>
123
124
125/*********************************************************************************************************************************
126* Structures and Typedefs *
127*********************************************************************************************************************************/
128/** @typedef PFNIEMOP
129 * Pointer to an opcode decoder function.
130 */
131
132/** @def FNIEMOP_DEF
133 * Define an opcode decoder function.
134 *
135 * We're using macors for this so that adding and removing parameters as well as
136 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL
137 *
138 * @param a_Name The function name.
139 */
140
141/** @typedef PFNIEMOPRM
142 * Pointer to an opcode decoder function with RM byte.
143 */
144
145/** @def FNIEMOPRM_DEF
146 * Define an opcode decoder function with RM byte.
147 *
148 * We're using macors for this so that adding and removing parameters as well as
149 * tweaking compiler specific attributes becomes easier. See FNIEMOP_CALL_1
150 *
151 * @param a_Name The function name.
152 */
153
154#if defined(__GNUC__) && defined(RT_ARCH_X86)
155typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOP)(PVMCPU pVCpu);
156typedef VBOXSTRICTRC (__attribute__((__fastcall__)) * PFNIEMOPRM)(PVMCPU pVCpu, uint8_t bRm);
157# define FNIEMOP_DEF(a_Name) \
158 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPU pVCpu)
159# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
160 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPU pVCpu, a_Type0 a_Name0)
161# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
162 IEM_STATIC VBOXSTRICTRC __attribute__((__fastcall__, __nothrow__)) a_Name(PVMCPU pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
163
164#elif defined(_MSC_VER) && defined(RT_ARCH_X86)
165typedef VBOXSTRICTRC (__fastcall * PFNIEMOP)(PVMCPU pVCpu);
166typedef VBOXSTRICTRC (__fastcall * PFNIEMOPRM)(PVMCPU pVCpu, uint8_t bRm);
167# define FNIEMOP_DEF(a_Name) \
168 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPU pVCpu) RT_NO_THROW_DEF
169# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
170 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPU pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
171# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
172 IEM_STATIC /*__declspec(naked)*/ VBOXSTRICTRC __fastcall a_Name(PVMCPU pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
173
174#elif defined(__GNUC__)
175typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPU pVCpu);
176typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPU pVCpu, uint8_t bRm);
177# define FNIEMOP_DEF(a_Name) \
178 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu)
179# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
180 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, a_Type0 a_Name0)
181# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
182 IEM_STATIC VBOXSTRICTRC __attribute__((__nothrow__)) a_Name(PVMCPU pVCpu, a_Type0 a_Name0, a_Type1 a_Name1)
183
184#else
185typedef VBOXSTRICTRC (* PFNIEMOP)(PVMCPU pVCpu);
186typedef VBOXSTRICTRC (* PFNIEMOPRM)(PVMCPU pVCpu, uint8_t bRm);
187# define FNIEMOP_DEF(a_Name) \
188 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPU pVCpu) RT_NO_THROW_DEF
189# define FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
190 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPU pVCpu, a_Type0 a_Name0) RT_NO_THROW_DEF
191# define FNIEMOP_DEF_2(a_Name, a_Type0, a_Name0, a_Type1, a_Name1) \
192 IEM_STATIC VBOXSTRICTRC a_Name(PVMCPU pVCpu, a_Type0 a_Name0, a_Type1 a_Name1) RT_NO_THROW_DEF
193
194#endif
195#define FNIEMOPRM_DEF(a_Name) FNIEMOP_DEF_1(a_Name, uint8_t, bRm)
196
197
198/**
199 * Selector descriptor table entry as fetched by iemMemFetchSelDesc.
200 */
201typedef union IEMSELDESC
202{
203 /** The legacy view. */
204 X86DESC Legacy;
205 /** The long mode view. */
206 X86DESC64 Long;
207} IEMSELDESC;
208/** Pointer to a selector descriptor table entry. */
209typedef IEMSELDESC *PIEMSELDESC;
210
211/**
212 * CPU exception classes.
213 */
214typedef enum IEMXCPTCLASS
215{
216 IEMXCPTCLASS_BENIGN,
217 IEMXCPTCLASS_CONTRIBUTORY,
218 IEMXCPTCLASS_PAGE_FAULT,
219 IEMXCPTCLASS_DOUBLE_FAULT
220} IEMXCPTCLASS;
221
222
223/*********************************************************************************************************************************
224* Defined Constants And Macros *
225*********************************************************************************************************************************/
226/** @def IEM_WITH_SETJMP
227 * Enables alternative status code handling using setjmps.
228 *
229 * This adds a bit of expense via the setjmp() call since it saves all the
230 * non-volatile registers. However, it eliminates return code checks and allows
231 * for more optimal return value passing (return regs instead of stack buffer).
232 */
233#if defined(DOXYGEN_RUNNING) || defined(RT_OS_WINDOWS) || 1
234# define IEM_WITH_SETJMP
235#endif
236
237/** Used to shut up GCC warnings about variables that 'may be used uninitialized'
238 * due to GCC lacking knowledge about the value range of a switch. */
239#define IEM_NOT_REACHED_DEFAULT_CASE_RET() default: AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE)
240
241/** Variant of IEM_NOT_REACHED_DEFAULT_CASE_RET that returns a custom value. */
242#define IEM_NOT_REACHED_DEFAULT_CASE_RET2(a_RetValue) default: AssertFailedReturn(a_RetValue)
243
244/**
245 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
246 * occation.
247 */
248#ifdef LOG_ENABLED
249# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
250 do { \
251 /*Log*/ LogAlways(("%s: returning IEM_RETURN_ASPECT_NOT_IMPLEMENTED (line %d)\n", __FUNCTION__, __LINE__)); \
252 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
253 } while (0)
254#else
255# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED() \
256 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
257#endif
258
259/**
260 * Returns IEM_RETURN_ASPECT_NOT_IMPLEMENTED, and in debug builds logs the
261 * occation using the supplied logger statement.
262 *
263 * @param a_LoggerArgs What to log on failure.
264 */
265#ifdef LOG_ENABLED
266# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
267 do { \
268 LogAlways((LOG_FN_FMT ": ", __PRETTY_FUNCTION__)); LogAlways(a_LoggerArgs); \
269 /*LogFunc(a_LoggerArgs);*/ \
270 return VERR_IEM_ASPECT_NOT_IMPLEMENTED; \
271 } while (0)
272#else
273# define IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(a_LoggerArgs) \
274 return VERR_IEM_ASPECT_NOT_IMPLEMENTED
275#endif
276
277/**
278 * Call an opcode decoder function.
279 *
280 * We're using macors for this so that adding and removing parameters can be
281 * done as we please. See FNIEMOP_DEF.
282 */
283#define FNIEMOP_CALL(a_pfn) (a_pfn)(pVCpu)
284
285/**
286 * Call a common opcode decoder function taking one extra argument.
287 *
288 * We're using macors for this so that adding and removing parameters can be
289 * done as we please. See FNIEMOP_DEF_1.
290 */
291#define FNIEMOP_CALL_1(a_pfn, a0) (a_pfn)(pVCpu, a0)
292
293/**
294 * Call a common opcode decoder function taking one extra argument.
295 *
296 * We're using macors for this so that adding and removing parameters can be
297 * done as we please. See FNIEMOP_DEF_1.
298 */
299#define FNIEMOP_CALL_2(a_pfn, a0, a1) (a_pfn)(pVCpu, a0, a1)
300
301/**
302 * Check if we're currently executing in real or virtual 8086 mode.
303 *
304 * @returns @c true if it is, @c false if not.
305 * @param a_pVCpu The IEM state of the current CPU.
306 */
307#define IEM_IS_REAL_OR_V86_MODE(a_pVCpu) (CPUMIsGuestInRealOrV86ModeEx(IEM_GET_CTX(a_pVCpu)))
308
309/**
310 * Check if we're currently executing in virtual 8086 mode.
311 *
312 * @returns @c true if it is, @c false if not.
313 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
314 */
315#define IEM_IS_V86_MODE(a_pVCpu) (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(a_pVCpu)))
316
317/**
318 * Check if we're currently executing in long mode.
319 *
320 * @returns @c true if it is, @c false if not.
321 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
322 */
323#define IEM_IS_LONG_MODE(a_pVCpu) (CPUMIsGuestInLongModeEx(IEM_GET_CTX(a_pVCpu)))
324
325/**
326 * Check if we're currently executing in a 64-bit code segment.
327 *
328 * @returns @c true if it is, @c false if not.
329 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
330 */
331#define IEM_IS_64BIT_CODE(a_pVCpu) (CPUMIsGuestIn64BitCodeEx(IEM_GET_CTX(a_pVCpu)))
332
333/**
334 * Check if we're currently executing in real mode.
335 *
336 * @returns @c true if it is, @c false if not.
337 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
338 */
339#define IEM_IS_REAL_MODE(a_pVCpu) (CPUMIsGuestInRealModeEx(IEM_GET_CTX(a_pVCpu)))
340
341/**
342 * Returns a (const) pointer to the CPUMFEATURES for the guest CPU.
343 * @returns PCCPUMFEATURES
344 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
345 */
346#define IEM_GET_GUEST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.GuestFeatures))
347
348/**
349 * Returns a (const) pointer to the CPUMFEATURES for the host CPU.
350 * @returns PCCPUMFEATURES
351 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
352 */
353#define IEM_GET_HOST_CPU_FEATURES(a_pVCpu) (&((a_pVCpu)->CTX_SUFF(pVM)->cpum.ro.HostFeatures))
354
355/**
356 * Evaluates to true if we're presenting an Intel CPU to the guest.
357 */
358#define IEM_IS_GUEST_CPU_INTEL(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL )
359
360/**
361 * Evaluates to true if we're presenting an AMD CPU to the guest.
362 */
363#define IEM_IS_GUEST_CPU_AMD(a_pVCpu) ( (a_pVCpu)->iem.s.enmCpuVendor == CPUMCPUVENDOR_AMD )
364
365/**
366 * Check if the address is canonical.
367 */
368#define IEM_IS_CANONICAL(a_u64Addr) X86_IS_CANONICAL(a_u64Addr)
369
370/**
371 * Gets the effective VEX.VVVV value.
372 *
373 * The 4th bit is ignored if not 64-bit code.
374 * @returns effective V-register value.
375 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
376 */
377#define IEM_GET_EFFECTIVE_VVVV(a_pVCpu) \
378 ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT ? (a_pVCpu)->iem.s.uVex3rdReg : (a_pVCpu)->iem.s.uVex3rdReg & 7)
379
380/** @def IEM_USE_UNALIGNED_DATA_ACCESS
381 * Use unaligned accesses instead of elaborate byte assembly. */
382#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86) || defined(DOXYGEN_RUNNING)
383# define IEM_USE_UNALIGNED_DATA_ACCESS
384#endif
385
386#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
387/**
388 * Check the common VMX instruction preconditions.
389 */
390#define IEM_VMX_INSTR_COMMON_CHECKS(a_pVCpu, a_szInstr, a_InsDiagPrefix) \
391 do { \
392 if (!IEM_IS_VMX_ENABLED(a_pVCpu)) \
393 { \
394 Log((a_szInstr ": CR4.VMXE not enabled -> #UD\n")); \
395 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmInstrDiag = a_InsDiagPrefix##_Vmxe; \
396 return iemRaiseUndefinedOpcode(a_pVCpu); \
397 } \
398 if (IEM_IS_REAL_OR_V86_MODE(a_pVCpu)) \
399 { \
400 Log((a_szInstr ": Real or v8086 mode -> #UD\n")); \
401 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmInstrDiag = a_InsDiagPrefix##_RealOrV86Mode; \
402 return iemRaiseUndefinedOpcode(a_pVCpu); \
403 } \
404 if (IEM_IS_LONG_MODE(a_pVCpu) && !IEM_IS_64BIT_CODE(a_pVCpu)) \
405 { \
406 Log((a_szInstr ": Long mode without 64-bit code segment -> #UD\n")); \
407 (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmInstrDiag = a_InsDiagPrefix##_LongModeCS; \
408 return iemRaiseUndefinedOpcode(a_pVCpu); \
409 } \
410 } while (0)
411
412/**
413 * Check if VMX is enabled.
414 */
415# define IEM_IS_VMX_ENABLED(a_pVCpu) (CPUMIsGuestVmxEnabled(IEM_GET_CTX(a_pVCpu)))
416
417/**
418 * Check if the guest has entered VMX root operation.
419 */
420#define IEM_IS_VMX_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxRootMode(IEM_GET_CTX(pVCpu)))
421
422/**
423 * Check if the guest has entered VMX non-root operation.
424 */
425#define IEM_IS_VMX_NON_ROOT_MODE(a_pVCpu) (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
426
427#else
428# define IEM_VMX_INSTR_COMMON_CHECKS(a_pVCpu, a_szInstr, a_InsDiagPrefix) do { } while (0)
429# define IEM_IS_VMX_ENABLED(a_pVCpu) (false)
430# define IEM_IS_VMX_ROOT_MODE(a_pVCpu) (false)
431# define IEM_IS_VMX_NON_ROOT_MODE(a_pVCpu) (false)
432
433#endif
434
435#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
436/**
437 * Check the common SVM instruction preconditions.
438 */
439# define IEM_SVM_INSTR_COMMON_CHECKS(a_pVCpu, a_Instr) \
440 do { \
441 if (!IEM_IS_SVM_ENABLED(a_pVCpu)) \
442 { \
443 Log((RT_STR(a_Instr) ": EFER.SVME not enabled -> #UD\n")); \
444 return iemRaiseUndefinedOpcode(a_pVCpu); \
445 } \
446 if (IEM_IS_REAL_OR_V86_MODE(a_pVCpu)) \
447 { \
448 Log((RT_STR(a_Instr) ": Real or v8086 mode -> #UD\n")); \
449 return iemRaiseUndefinedOpcode(a_pVCpu); \
450 } \
451 if ((a_pVCpu)->iem.s.uCpl != 0) \
452 { \
453 Log((RT_STR(a_Instr) ": CPL != 0 -> #GP(0)\n")); \
454 return iemRaiseGeneralProtectionFault0(a_pVCpu); \
455 } \
456 } while (0)
457
458/**
459 * Updates the NextRIP (NRI) field in the nested-guest VMCB.
460 */
461# define IEM_SVM_UPDATE_NRIP(a_pVCpu) \
462 do { \
463 if (IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmNextRipSave) \
464 CPUMGuestSvmUpdateNRip(a_pVCpu, IEM_GET_CTX(a_pVCpu), IEM_GET_INSTR_LEN(a_pVCpu)); \
465 } while (0)
466
467/**
468 * Check if SVM is enabled.
469 */
470# define IEM_IS_SVM_ENABLED(a_pVCpu) (CPUMIsGuestSvmEnabled(IEM_GET_CTX(a_pVCpu)))
471
472/**
473 * Check if an SVM control/instruction intercept is set.
474 */
475# define IEM_IS_SVM_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (CPUMIsGuestSvmCtrlInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_Intercept)))
476
477/**
478 * Check if an SVM read CRx intercept is set.
479 */
480# define IEM_IS_SVM_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (CPUMIsGuestSvmReadCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
481
482/**
483 * Check if an SVM write CRx intercept is set.
484 */
485# define IEM_IS_SVM_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (CPUMIsGuestSvmWriteCRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uCr)))
486
487/**
488 * Check if an SVM read DRx intercept is set.
489 */
490# define IEM_IS_SVM_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (CPUMIsGuestSvmReadDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
491
492/**
493 * Check if an SVM write DRx intercept is set.
494 */
495# define IEM_IS_SVM_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (CPUMIsGuestSvmWriteDRxInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uDr)))
496
497/**
498 * Check if an SVM exception intercept is set.
499 */
500# define IEM_IS_SVM_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (CPUMIsGuestSvmXcptInterceptSet(a_pVCpu, IEM_GET_CTX(a_pVCpu), (a_uVector)))
501
502/**
503 * Get the SVM pause-filter count.
504 */
505# define IEM_GET_SVM_PAUSE_FILTER_COUNT(a_pVCpu) (CPUMGetGuestSvmPauseFilterCount(a_pVCpu, IEM_GET_CTX(a_pVCpu)))
506
507/**
508 * Invokes the SVM \#VMEXIT handler for the nested-guest.
509 */
510# define IEM_RETURN_SVM_VMEXIT(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
511 do { return iemSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); } while (0)
512
513/**
514 * Invokes the 'MOV CRx' SVM \#VMEXIT handler after constructing the
515 * corresponding decode assist information.
516 */
517# define IEM_RETURN_SVM_CRX_VMEXIT(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) \
518 do \
519 { \
520 uint64_t uExitInfo1; \
521 if ( IEM_GET_GUEST_CPU_FEATURES(a_pVCpu)->fSvmDecodeAssists \
522 && (a_enmAccessCrX) == IEMACCESSCRX_MOV_CRX) \
523 uExitInfo1 = SVM_EXIT1_MOV_CRX_MASK | ((a_iGReg) & 7); \
524 else \
525 uExitInfo1 = 0; \
526 IEM_RETURN_SVM_VMEXIT(a_pVCpu, a_uExitCode, uExitInfo1, 0); \
527 } while (0)
528
529#else
530# define IEM_SVM_INSTR_COMMON_CHECKS(a_pVCpu, a_Instr) do { } while (0)
531# define IEM_SVM_UPDATE_NRIP(a_pVCpu) do { } while (0)
532# define IEM_IS_SVM_ENABLED(a_pVCpu) (false)
533# define IEM_IS_SVM_CTRL_INTERCEPT_SET(a_pVCpu, a_Intercept) (false)
534# define IEM_IS_SVM_READ_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
535# define IEM_IS_SVM_WRITE_CR_INTERCEPT_SET(a_pVCpu, a_uCr) (false)
536# define IEM_IS_SVM_READ_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
537# define IEM_IS_SVM_WRITE_DR_INTERCEPT_SET(a_pVCpu, a_uDr) (false)
538# define IEM_IS_SVM_XCPT_INTERCEPT_SET(a_pVCpu, a_uVector) (false)
539# define IEM_GET_SVM_PAUSE_FILTER_COUNT(a_pVCpu) (0)
540# define IEM_RETURN_SVM_VMEXIT(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) do { return VERR_SVM_IPE_1; } while (0)
541# define IEM_RETURN_SVM_CRX_VMEXIT(a_pVCpu, a_uExitCode, a_enmAccessCrX, a_iGReg) do { return VERR_SVM_IPE_1; } while (0)
542
543#endif
544
545
546/*********************************************************************************************************************************
547* Global Variables *
548*********************************************************************************************************************************/
549extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
550
551
552/** Function table for the ADD instruction. */
553IEM_STATIC const IEMOPBINSIZES g_iemAImpl_add =
554{
555 iemAImpl_add_u8, iemAImpl_add_u8_locked,
556 iemAImpl_add_u16, iemAImpl_add_u16_locked,
557 iemAImpl_add_u32, iemAImpl_add_u32_locked,
558 iemAImpl_add_u64, iemAImpl_add_u64_locked
559};
560
561/** Function table for the ADC instruction. */
562IEM_STATIC const IEMOPBINSIZES g_iemAImpl_adc =
563{
564 iemAImpl_adc_u8, iemAImpl_adc_u8_locked,
565 iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
566 iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
567 iemAImpl_adc_u64, iemAImpl_adc_u64_locked
568};
569
570/** Function table for the SUB instruction. */
571IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sub =
572{
573 iemAImpl_sub_u8, iemAImpl_sub_u8_locked,
574 iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
575 iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
576 iemAImpl_sub_u64, iemAImpl_sub_u64_locked
577};
578
579/** Function table for the SBB instruction. */
580IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sbb =
581{
582 iemAImpl_sbb_u8, iemAImpl_sbb_u8_locked,
583 iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
584 iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
585 iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
586};
587
588/** Function table for the OR instruction. */
589IEM_STATIC const IEMOPBINSIZES g_iemAImpl_or =
590{
591 iemAImpl_or_u8, iemAImpl_or_u8_locked,
592 iemAImpl_or_u16, iemAImpl_or_u16_locked,
593 iemAImpl_or_u32, iemAImpl_or_u32_locked,
594 iemAImpl_or_u64, iemAImpl_or_u64_locked
595};
596
597/** Function table for the XOR instruction. */
598IEM_STATIC const IEMOPBINSIZES g_iemAImpl_xor =
599{
600 iemAImpl_xor_u8, iemAImpl_xor_u8_locked,
601 iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
602 iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
603 iemAImpl_xor_u64, iemAImpl_xor_u64_locked
604};
605
606/** Function table for the AND instruction. */
607IEM_STATIC const IEMOPBINSIZES g_iemAImpl_and =
608{
609 iemAImpl_and_u8, iemAImpl_and_u8_locked,
610 iemAImpl_and_u16, iemAImpl_and_u16_locked,
611 iemAImpl_and_u32, iemAImpl_and_u32_locked,
612 iemAImpl_and_u64, iemAImpl_and_u64_locked
613};
614
615/** Function table for the CMP instruction.
616 * @remarks Making operand order ASSUMPTIONS.
617 */
618IEM_STATIC const IEMOPBINSIZES g_iemAImpl_cmp =
619{
620 iemAImpl_cmp_u8, NULL,
621 iemAImpl_cmp_u16, NULL,
622 iemAImpl_cmp_u32, NULL,
623 iemAImpl_cmp_u64, NULL
624};
625
626/** Function table for the TEST instruction.
627 * @remarks Making operand order ASSUMPTIONS.
628 */
629IEM_STATIC const IEMOPBINSIZES g_iemAImpl_test =
630{
631 iemAImpl_test_u8, NULL,
632 iemAImpl_test_u16, NULL,
633 iemAImpl_test_u32, NULL,
634 iemAImpl_test_u64, NULL
635};
636
637/** Function table for the BT instruction. */
638IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bt =
639{
640 NULL, NULL,
641 iemAImpl_bt_u16, NULL,
642 iemAImpl_bt_u32, NULL,
643 iemAImpl_bt_u64, NULL
644};
645
646/** Function table for the BTC instruction. */
647IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btc =
648{
649 NULL, NULL,
650 iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
651 iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
652 iemAImpl_btc_u64, iemAImpl_btc_u64_locked
653};
654
655/** Function table for the BTR instruction. */
656IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btr =
657{
658 NULL, NULL,
659 iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
660 iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
661 iemAImpl_btr_u64, iemAImpl_btr_u64_locked
662};
663
664/** Function table for the BTS instruction. */
665IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bts =
666{
667 NULL, NULL,
668 iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
669 iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
670 iemAImpl_bts_u64, iemAImpl_bts_u64_locked
671};
672
673/** Function table for the BSF instruction. */
674IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf =
675{
676 NULL, NULL,
677 iemAImpl_bsf_u16, NULL,
678 iemAImpl_bsf_u32, NULL,
679 iemAImpl_bsf_u64, NULL
680};
681
682/** Function table for the BSR instruction. */
683IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr =
684{
685 NULL, NULL,
686 iemAImpl_bsr_u16, NULL,
687 iemAImpl_bsr_u32, NULL,
688 iemAImpl_bsr_u64, NULL
689};
690
691/** Function table for the IMUL instruction. */
692IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two =
693{
694 NULL, NULL,
695 iemAImpl_imul_two_u16, NULL,
696 iemAImpl_imul_two_u32, NULL,
697 iemAImpl_imul_two_u64, NULL
698};
699
700/** Group 1 /r lookup table. */
701IEM_STATIC const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
702{
703 &g_iemAImpl_add,
704 &g_iemAImpl_or,
705 &g_iemAImpl_adc,
706 &g_iemAImpl_sbb,
707 &g_iemAImpl_and,
708 &g_iemAImpl_sub,
709 &g_iemAImpl_xor,
710 &g_iemAImpl_cmp
711};
712
713/** Function table for the INC instruction. */
714IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_inc =
715{
716 iemAImpl_inc_u8, iemAImpl_inc_u8_locked,
717 iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
718 iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
719 iemAImpl_inc_u64, iemAImpl_inc_u64_locked
720};
721
722/** Function table for the DEC instruction. */
723IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_dec =
724{
725 iemAImpl_dec_u8, iemAImpl_dec_u8_locked,
726 iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
727 iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
728 iemAImpl_dec_u64, iemAImpl_dec_u64_locked
729};
730
731/** Function table for the NEG instruction. */
732IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_neg =
733{
734 iemAImpl_neg_u8, iemAImpl_neg_u8_locked,
735 iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
736 iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
737 iemAImpl_neg_u64, iemAImpl_neg_u64_locked
738};
739
740/** Function table for the NOT instruction. */
741IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_not =
742{
743 iemAImpl_not_u8, iemAImpl_not_u8_locked,
744 iemAImpl_not_u16, iemAImpl_not_u16_locked,
745 iemAImpl_not_u32, iemAImpl_not_u32_locked,
746 iemAImpl_not_u64, iemAImpl_not_u64_locked
747};
748
749
750/** Function table for the ROL instruction. */
751IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol =
752{
753 iemAImpl_rol_u8,
754 iemAImpl_rol_u16,
755 iemAImpl_rol_u32,
756 iemAImpl_rol_u64
757};
758
759/** Function table for the ROR instruction. */
760IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror =
761{
762 iemAImpl_ror_u8,
763 iemAImpl_ror_u16,
764 iemAImpl_ror_u32,
765 iemAImpl_ror_u64
766};
767
768/** Function table for the RCL instruction. */
769IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl =
770{
771 iemAImpl_rcl_u8,
772 iemAImpl_rcl_u16,
773 iemAImpl_rcl_u32,
774 iemAImpl_rcl_u64
775};
776
777/** Function table for the RCR instruction. */
778IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr =
779{
780 iemAImpl_rcr_u8,
781 iemAImpl_rcr_u16,
782 iemAImpl_rcr_u32,
783 iemAImpl_rcr_u64
784};
785
786/** Function table for the SHL instruction. */
787IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl =
788{
789 iemAImpl_shl_u8,
790 iemAImpl_shl_u16,
791 iemAImpl_shl_u32,
792 iemAImpl_shl_u64
793};
794
795/** Function table for the SHR instruction. */
796IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr =
797{
798 iemAImpl_shr_u8,
799 iemAImpl_shr_u16,
800 iemAImpl_shr_u32,
801 iemAImpl_shr_u64
802};
803
804/** Function table for the SAR instruction. */
805IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar =
806{
807 iemAImpl_sar_u8,
808 iemAImpl_sar_u16,
809 iemAImpl_sar_u32,
810 iemAImpl_sar_u64
811};
812
813
814/** Function table for the MUL instruction. */
815IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul =
816{
817 iemAImpl_mul_u8,
818 iemAImpl_mul_u16,
819 iemAImpl_mul_u32,
820 iemAImpl_mul_u64
821};
822
823/** Function table for the IMUL instruction working implicitly on rAX. */
824IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul =
825{
826 iemAImpl_imul_u8,
827 iemAImpl_imul_u16,
828 iemAImpl_imul_u32,
829 iemAImpl_imul_u64
830};
831
832/** Function table for the DIV instruction. */
833IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div =
834{
835 iemAImpl_div_u8,
836 iemAImpl_div_u16,
837 iemAImpl_div_u32,
838 iemAImpl_div_u64
839};
840
841/** Function table for the MUL instruction. */
842IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv =
843{
844 iemAImpl_idiv_u8,
845 iemAImpl_idiv_u16,
846 iemAImpl_idiv_u32,
847 iemAImpl_idiv_u64
848};
849
850/** Function table for the SHLD instruction */
851IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
852{
853 iemAImpl_shld_u16,
854 iemAImpl_shld_u32,
855 iemAImpl_shld_u64,
856};
857
858/** Function table for the SHRD instruction */
859IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
860{
861 iemAImpl_shrd_u16,
862 iemAImpl_shrd_u32,
863 iemAImpl_shrd_u64,
864};
865
866
867/** Function table for the PUNPCKLBW instruction */
868IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklbw = { iemAImpl_punpcklbw_u64, iemAImpl_punpcklbw_u128 };
869/** Function table for the PUNPCKLBD instruction */
870IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklwd = { iemAImpl_punpcklwd_u64, iemAImpl_punpcklwd_u128 };
871/** Function table for the PUNPCKLDQ instruction */
872IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpckldq = { iemAImpl_punpckldq_u64, iemAImpl_punpckldq_u128 };
873/** Function table for the PUNPCKLQDQ instruction */
874IEM_STATIC const IEMOPMEDIAF1L1 g_iemAImpl_punpcklqdq = { NULL, iemAImpl_punpcklqdq_u128 };
875
876/** Function table for the PUNPCKHBW instruction */
877IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhbw = { iemAImpl_punpckhbw_u64, iemAImpl_punpckhbw_u128 };
878/** Function table for the PUNPCKHBD instruction */
879IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhwd = { iemAImpl_punpckhwd_u64, iemAImpl_punpckhwd_u128 };
880/** Function table for the PUNPCKHDQ instruction */
881IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhdq = { iemAImpl_punpckhdq_u64, iemAImpl_punpckhdq_u128 };
882/** Function table for the PUNPCKHQDQ instruction */
883IEM_STATIC const IEMOPMEDIAF1H1 g_iemAImpl_punpckhqdq = { NULL, iemAImpl_punpckhqdq_u128 };
884
885/** Function table for the PXOR instruction */
886IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pxor = { iemAImpl_pxor_u64, iemAImpl_pxor_u128 };
887/** Function table for the PCMPEQB instruction */
888IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqb = { iemAImpl_pcmpeqb_u64, iemAImpl_pcmpeqb_u128 };
889/** Function table for the PCMPEQW instruction */
890IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqw = { iemAImpl_pcmpeqw_u64, iemAImpl_pcmpeqw_u128 };
891/** Function table for the PCMPEQD instruction */
892IEM_STATIC const IEMOPMEDIAF2 g_iemAImpl_pcmpeqd = { iemAImpl_pcmpeqd_u64, iemAImpl_pcmpeqd_u128 };
893
894
895#if defined(IEM_LOG_MEMORY_WRITES)
896/** What IEM just wrote. */
897uint8_t g_abIemWrote[256];
898/** How much IEM just wrote. */
899size_t g_cbIemWrote;
900#endif
901
902
903/*********************************************************************************************************************************
904* Internal Functions *
905*********************************************************************************************************************************/
906IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPU pVCpu, uint16_t uErr);
907IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPU pVCpu);
908IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPU pVCpu);
909IEM_STATIC VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPU pVCpu, uint16_t uSel);
910/*IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresent(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess);*/
911IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPU pVCpu, uint16_t uSel);
912IEM_STATIC VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPU pVCpu, uint16_t uErr);
913IEM_STATIC VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPU pVCpu, uint16_t uSel);
914IEM_STATIC VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPU pVCpu, uint16_t uErr);
915IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPU pVCpu, uint16_t uErr);
916IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPU pVCpu);
917IEM_STATIC VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPU pVCpu, RTSEL uSel);
918IEM_STATIC VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess);
919IEM_STATIC VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPU pVCpu, RTSEL Sel);
920IEM_STATIC VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess);
921IEM_STATIC VBOXSTRICTRC iemRaisePageFault(PVMCPU pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
922IEM_STATIC VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPU pVCpu);
923#ifdef IEM_WITH_SETJMP
924DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaisePageFaultJmp(PVMCPU pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc);
925DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPU pVCpu);
926DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess);
927DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPU pVCpu, RTSEL Sel);
928DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess);
929#endif
930
931IEM_STATIC VBOXSTRICTRC iemMemMap(PVMCPU pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess);
932IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPU pVCpu, void *pvMem, uint32_t fAccess);
933IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPU pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
934IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPU pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
935IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPU pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
936IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPU pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
937IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPU pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
938IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPU pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem);
939IEM_STATIC VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPU pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode);
940IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPU pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt);
941IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPU pVCpu, void *pvMem, uint64_t uNewRsp);
942IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPU pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp);
943IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPU pVCpu, uint32_t u32Value);
944IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPU pVCpu, uint16_t u16Value);
945IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPU pVCpu, uint16_t uSel);
946IEM_STATIC uint16_t iemSRegFetchU16(PVMCPU pVCpu, uint8_t iSegReg);
947IEM_STATIC uint64_t iemSRegBaseFetchU64(PVMCPU pVCpu, uint8_t iSegReg);
948
949#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
950IEM_STATIC VBOXSTRICTRC iemSvmVmexit(PVMCPU pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2);
951IEM_STATIC VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPU pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2);
952#endif
953
954#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
955IEM_STATIC VBOXSTRICTRC iemVmxVmxon(PVMCPU pVCpu, uint8_t cbInstr, RTGCPHYS GCPtrVmxon, PCVMXEXITINSTRINFO pExitInstrInfo,
956 RTGCPTR GCPtrDisp);
957#endif
958
959/**
960 * Sets the pass up status.
961 *
962 * @returns VINF_SUCCESS.
963 * @param pVCpu The cross context virtual CPU structure of the
964 * calling thread.
965 * @param rcPassUp The pass up status. Must be informational.
966 * VINF_SUCCESS is not allowed.
967 */
968IEM_STATIC int iemSetPassUpStatus(PVMCPU pVCpu, VBOXSTRICTRC rcPassUp)
969{
970 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
971
972 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
973 if (rcOldPassUp == VINF_SUCCESS)
974 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
975 /* If both are EM scheduling codes, use EM priority rules. */
976 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
977 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
978 {
979 if (rcPassUp < rcOldPassUp)
980 {
981 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
982 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
983 }
984 else
985 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
986 }
987 /* Override EM scheduling with specific status code. */
988 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
989 {
990 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
991 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
992 }
993 /* Don't override specific status code, first come first served. */
994 else
995 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
996 return VINF_SUCCESS;
997}
998
999
1000/**
1001 * Calculates the CPU mode.
1002 *
1003 * This is mainly for updating IEMCPU::enmCpuMode.
1004 *
1005 * @returns CPU mode.
1006 * @param pVCpu The cross context virtual CPU structure of the
1007 * calling thread.
1008 */
1009DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPU pVCpu)
1010{
1011 if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
1012 return IEMMODE_64BIT;
1013 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
1014 return IEMMODE_32BIT;
1015 return IEMMODE_16BIT;
1016}
1017
1018
1019/**
1020 * Initializes the execution state.
1021 *
1022 * @param pVCpu The cross context virtual CPU structure of the
1023 * calling thread.
1024 * @param fBypassHandlers Whether to bypass access handlers.
1025 *
1026 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
1027 * side-effects in strict builds.
1028 */
1029DECLINLINE(void) iemInitExec(PVMCPU pVCpu, bool fBypassHandlers)
1030{
1031 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
1032 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1033
1034#if defined(VBOX_STRICT) && !defined(VBOX_WITH_RAW_MODE_NOT_R0)
1035 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1036 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1037 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1038 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1039 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1040 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1041 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1042 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1043#endif
1044
1045#ifdef VBOX_WITH_RAW_MODE_NOT_R0
1046 CPUMGuestLazyLoadHiddenCsAndSs(pVCpu);
1047#endif
1048 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
1049 pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
1050#ifdef VBOX_STRICT
1051 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
1052 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
1053 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
1054 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
1055 pVCpu->iem.s.fPrefixes = 0xfeedbeef;
1056 pVCpu->iem.s.uRexReg = 127;
1057 pVCpu->iem.s.uRexB = 127;
1058 pVCpu->iem.s.offModRm = 127;
1059 pVCpu->iem.s.uRexIndex = 127;
1060 pVCpu->iem.s.iEffSeg = 127;
1061 pVCpu->iem.s.idxPrefix = 127;
1062 pVCpu->iem.s.uVex3rdReg = 127;
1063 pVCpu->iem.s.uVexLength = 127;
1064 pVCpu->iem.s.fEvexStuff = 127;
1065 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
1066# ifdef IEM_WITH_CODE_TLB
1067 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
1068 pVCpu->iem.s.pbInstrBuf = NULL;
1069 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
1070 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
1071 pVCpu->iem.s.offCurInstrStart = INT16_MAX;
1072 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
1073# else
1074 pVCpu->iem.s.offOpcode = 127;
1075 pVCpu->iem.s.cbOpcode = 127;
1076# endif
1077#endif
1078
1079 pVCpu->iem.s.cActiveMappings = 0;
1080 pVCpu->iem.s.iNextMapping = 0;
1081 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1082 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
1083#ifdef VBOX_WITH_RAW_MODE_NOT_R0
1084 pVCpu->iem.s.fInPatchCode = pVCpu->iem.s.uCpl == 0
1085 && pVCpu->cpum.GstCtx.cs.u64Base == 0
1086 && pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX
1087 && PATMIsPatchGCAddr(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.eip);
1088 if (!pVCpu->iem.s.fInPatchCode)
1089 CPUMRawLeave(pVCpu, VINF_SUCCESS);
1090#endif
1091}
1092
1093#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1094/**
1095 * Performs a minimal reinitialization of the execution state.
1096 *
1097 * This is intended to be used by VM-exits, SMM, LOADALL and other similar
1098 * 'world-switch' types operations on the CPU. Currently only nested
1099 * hardware-virtualization uses it.
1100 *
1101 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1102 */
1103IEM_STATIC void iemReInitExec(PVMCPU pVCpu)
1104{
1105 IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
1106 uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
1107
1108 pVCpu->iem.s.uCpl = uCpl;
1109 pVCpu->iem.s.enmCpuMode = enmMode;
1110 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1111 pVCpu->iem.s.enmEffAddrMode = enmMode;
1112 if (enmMode != IEMMODE_64BIT)
1113 {
1114 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1115 pVCpu->iem.s.enmEffOpSize = enmMode;
1116 }
1117 else
1118 {
1119 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1120 pVCpu->iem.s.enmEffOpSize = enmMode;
1121 }
1122 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1123#ifndef IEM_WITH_CODE_TLB
1124 /** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
1125 pVCpu->iem.s.offOpcode = 0;
1126 pVCpu->iem.s.cbOpcode = 0;
1127#endif
1128 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1129}
1130#endif
1131
1132/**
1133 * Counterpart to #iemInitExec that undoes evil strict-build stuff.
1134 *
1135 * @param pVCpu The cross context virtual CPU structure of the
1136 * calling thread.
1137 */
1138DECLINLINE(void) iemUninitExec(PVMCPU pVCpu)
1139{
1140 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
1141#ifdef VBOX_STRICT
1142# ifdef IEM_WITH_CODE_TLB
1143 NOREF(pVCpu);
1144# else
1145 pVCpu->iem.s.cbOpcode = 0;
1146# endif
1147#else
1148 NOREF(pVCpu);
1149#endif
1150}
1151
1152
1153/**
1154 * Initializes the decoder state.
1155 *
1156 * iemReInitDecoder is mostly a copy of this function.
1157 *
1158 * @param pVCpu The cross context virtual CPU structure of the
1159 * calling thread.
1160 * @param fBypassHandlers Whether to bypass access handlers.
1161 */
1162DECLINLINE(void) iemInitDecoder(PVMCPU pVCpu, bool fBypassHandlers)
1163{
1164 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
1165 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1166
1167#if defined(VBOX_STRICT) && !defined(VBOX_WITH_RAW_MODE_NOT_R0)
1168 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1169 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1170 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1171 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1172 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1173 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1174 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1175 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1176#endif
1177
1178#ifdef VBOX_WITH_RAW_MODE_NOT_R0
1179 CPUMGuestLazyLoadHiddenCsAndSs(pVCpu);
1180#endif
1181 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
1182 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
1183 pVCpu->iem.s.enmCpuMode = enmMode;
1184 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1185 pVCpu->iem.s.enmEffAddrMode = enmMode;
1186 if (enmMode != IEMMODE_64BIT)
1187 {
1188 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1189 pVCpu->iem.s.enmEffOpSize = enmMode;
1190 }
1191 else
1192 {
1193 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1194 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
1195 }
1196 pVCpu->iem.s.fPrefixes = 0;
1197 pVCpu->iem.s.uRexReg = 0;
1198 pVCpu->iem.s.uRexB = 0;
1199 pVCpu->iem.s.uRexIndex = 0;
1200 pVCpu->iem.s.idxPrefix = 0;
1201 pVCpu->iem.s.uVex3rdReg = 0;
1202 pVCpu->iem.s.uVexLength = 0;
1203 pVCpu->iem.s.fEvexStuff = 0;
1204 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1205#ifdef IEM_WITH_CODE_TLB
1206 pVCpu->iem.s.pbInstrBuf = NULL;
1207 pVCpu->iem.s.offInstrNextByte = 0;
1208 pVCpu->iem.s.offCurInstrStart = 0;
1209# ifdef VBOX_STRICT
1210 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
1211 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
1212 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
1213# endif
1214#else
1215 pVCpu->iem.s.offOpcode = 0;
1216 pVCpu->iem.s.cbOpcode = 0;
1217#endif
1218 pVCpu->iem.s.offModRm = 0;
1219 pVCpu->iem.s.cActiveMappings = 0;
1220 pVCpu->iem.s.iNextMapping = 0;
1221 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
1222 pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
1223#ifdef VBOX_WITH_RAW_MODE_NOT_R0
1224 pVCpu->iem.s.fInPatchCode = pVCpu->iem.s.uCpl == 0
1225 && pVCpu->cpum.GstCtx.cs.u64Base == 0
1226 && pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX
1227 && PATMIsPatchGCAddr(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.eip);
1228 if (!pVCpu->iem.s.fInPatchCode)
1229 CPUMRawLeave(pVCpu, VINF_SUCCESS);
1230#endif
1231
1232#ifdef DBGFTRACE_ENABLED
1233 switch (enmMode)
1234 {
1235 case IEMMODE_64BIT:
1236 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
1237 break;
1238 case IEMMODE_32BIT:
1239 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1240 break;
1241 case IEMMODE_16BIT:
1242 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1243 break;
1244 }
1245#endif
1246}
1247
1248
1249/**
1250 * Reinitializes the decoder state 2nd+ loop of IEMExecLots.
1251 *
1252 * This is mostly a copy of iemInitDecoder.
1253 *
1254 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
1255 */
1256DECLINLINE(void) iemReInitDecoder(PVMCPU pVCpu)
1257{
1258 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
1259
1260#if defined(VBOX_STRICT) && !defined(VBOX_WITH_RAW_MODE_NOT_R0)
1261 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
1262 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
1263 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
1264 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
1265 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
1266 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
1267 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
1268 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
1269#endif
1270
1271 pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu); /** @todo this should be updated during execution! */
1272 IEMMODE enmMode = iemCalcCpuMode(pVCpu);
1273 pVCpu->iem.s.enmCpuMode = enmMode; /** @todo this should be updated during execution! */
1274 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
1275 pVCpu->iem.s.enmEffAddrMode = enmMode;
1276 if (enmMode != IEMMODE_64BIT)
1277 {
1278 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
1279 pVCpu->iem.s.enmEffOpSize = enmMode;
1280 }
1281 else
1282 {
1283 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
1284 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
1285 }
1286 pVCpu->iem.s.fPrefixes = 0;
1287 pVCpu->iem.s.uRexReg = 0;
1288 pVCpu->iem.s.uRexB = 0;
1289 pVCpu->iem.s.uRexIndex = 0;
1290 pVCpu->iem.s.idxPrefix = 0;
1291 pVCpu->iem.s.uVex3rdReg = 0;
1292 pVCpu->iem.s.uVexLength = 0;
1293 pVCpu->iem.s.fEvexStuff = 0;
1294 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
1295#ifdef IEM_WITH_CODE_TLB
1296 if (pVCpu->iem.s.pbInstrBuf)
1297 {
1298 uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rip : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
1299 - pVCpu->iem.s.uInstrBufPc;
1300 if (off < pVCpu->iem.s.cbInstrBufTotal)
1301 {
1302 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1303 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1304 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1305 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1306 else
1307 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1308 }
1309 else
1310 {
1311 pVCpu->iem.s.pbInstrBuf = NULL;
1312 pVCpu->iem.s.offInstrNextByte = 0;
1313 pVCpu->iem.s.offCurInstrStart = 0;
1314 pVCpu->iem.s.cbInstrBuf = 0;
1315 pVCpu->iem.s.cbInstrBufTotal = 0;
1316 }
1317 }
1318 else
1319 {
1320 pVCpu->iem.s.offInstrNextByte = 0;
1321 pVCpu->iem.s.offCurInstrStart = 0;
1322 pVCpu->iem.s.cbInstrBuf = 0;
1323 pVCpu->iem.s.cbInstrBufTotal = 0;
1324 }
1325#else
1326 pVCpu->iem.s.cbOpcode = 0;
1327 pVCpu->iem.s.offOpcode = 0;
1328#endif
1329 pVCpu->iem.s.offModRm = 0;
1330 Assert(pVCpu->iem.s.cActiveMappings == 0);
1331 pVCpu->iem.s.iNextMapping = 0;
1332 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
1333 Assert(pVCpu->iem.s.fBypassHandlers == false);
1334#ifdef VBOX_WITH_RAW_MODE_NOT_R0
1335 if (!pVCpu->iem.s.fInPatchCode)
1336 { /* likely */ }
1337 else
1338 {
1339 pVCpu->iem.s.fInPatchCode = pVCpu->iem.s.uCpl == 0
1340 && pVCpu->cpum.GstCtx.cs.u64Base == 0
1341 && pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX
1342 && PATMIsPatchGCAddr(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.eip);
1343 if (!pVCpu->iem.s.fInPatchCode)
1344 CPUMRawLeave(pVCpu, VINF_SUCCESS);
1345 }
1346#endif
1347
1348#ifdef DBGFTRACE_ENABLED
1349 switch (enmMode)
1350 {
1351 case IEMMODE_64BIT:
1352 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
1353 break;
1354 case IEMMODE_32BIT:
1355 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1356 break;
1357 case IEMMODE_16BIT:
1358 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
1359 break;
1360 }
1361#endif
1362}
1363
1364
1365
1366/**
1367 * Prefetch opcodes the first time when starting executing.
1368 *
1369 * @returns Strict VBox status code.
1370 * @param pVCpu The cross context virtual CPU structure of the
1371 * calling thread.
1372 * @param fBypassHandlers Whether to bypass access handlers.
1373 */
1374IEM_STATIC VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPU pVCpu, bool fBypassHandlers)
1375{
1376 iemInitDecoder(pVCpu, fBypassHandlers);
1377
1378#ifdef IEM_WITH_CODE_TLB
1379 /** @todo Do ITLB lookup here. */
1380
1381#else /* !IEM_WITH_CODE_TLB */
1382
1383 /*
1384 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1385 *
1386 * First translate CS:rIP to a physical address.
1387 */
1388 uint32_t cbToTryRead;
1389 RTGCPTR GCPtrPC;
1390 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1391 {
1392 cbToTryRead = PAGE_SIZE;
1393 GCPtrPC = pVCpu->cpum.GstCtx.rip;
1394 if (IEM_IS_CANONICAL(GCPtrPC))
1395 cbToTryRead = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
1396 else
1397 return iemRaiseGeneralProtectionFault0(pVCpu);
1398 }
1399 else
1400 {
1401 uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
1402 AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1403 if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
1404 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
1405 else
1406 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1407 if (cbToTryRead) { /* likely */ }
1408 else /* overflowed */
1409 {
1410 Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1411 cbToTryRead = UINT32_MAX;
1412 }
1413 GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
1414 Assert(GCPtrPC <= UINT32_MAX);
1415 }
1416
1417# ifdef VBOX_WITH_RAW_MODE_NOT_R0
1418 /* Allow interpretation of patch manager code blocks since they can for
1419 instance throw #PFs for perfectly good reasons. */
1420 if (pVCpu->iem.s.fInPatchCode)
1421 {
1422 size_t cbRead = 0;
1423 int rc = PATMReadPatchCode(pVCpu->CTX_SUFF(pVM), GCPtrPC, pVCpu->iem.s.abOpcode, sizeof(pVCpu->iem.s.abOpcode), &cbRead);
1424 AssertRCReturn(rc, rc);
1425 pVCpu->iem.s.cbOpcode = (uint8_t)cbRead; Assert(pVCpu->iem.s.cbOpcode == cbRead); Assert(cbRead > 0);
1426 return VINF_SUCCESS;
1427 }
1428# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
1429
1430 RTGCPHYS GCPhys;
1431 uint64_t fFlags;
1432 int rc = PGMGstGetPage(pVCpu, GCPtrPC, &fFlags, &GCPhys);
1433 if (RT_SUCCESS(rc)) { /* probable */ }
1434 else
1435 {
1436 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
1437 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, rc);
1438 }
1439 if ((fFlags & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ }
1440 else
1441 {
1442 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
1443 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1444 }
1445 if (!(fFlags & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
1446 else
1447 {
1448 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
1449 return iemRaisePageFault(pVCpu, GCPtrPC, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1450 }
1451 GCPhys |= GCPtrPC & PAGE_OFFSET_MASK;
1452 /** @todo Check reserved bits and such stuff. PGM is better at doing
1453 * that, so do it when implementing the guest virtual address
1454 * TLB... */
1455
1456 /*
1457 * Read the bytes at this address.
1458 */
1459 PVM pVM = pVCpu->CTX_SUFF(pVM);
1460# if defined(IN_RING3) && defined(VBOX_WITH_RAW_MODE_NOT_R0)
1461 size_t cbActual;
1462 if ( PATMIsEnabled(pVM)
1463 && RT_SUCCESS(PATMR3ReadOrgInstr(pVM, GCPtrPC, pVCpu->iem.s.abOpcode, sizeof(pVCpu->iem.s.abOpcode), &cbActual)))
1464 {
1465 Log4(("decode - Read %u unpatched bytes at %RGv\n", cbActual, GCPtrPC));
1466 Assert(cbActual > 0);
1467 pVCpu->iem.s.cbOpcode = (uint8_t)cbActual;
1468 }
1469 else
1470# endif
1471 {
1472 uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrPC & PAGE_OFFSET_MASK);
1473 if (cbToTryRead > cbLeftOnPage)
1474 cbToTryRead = cbLeftOnPage;
1475 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
1476 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
1477
1478 if (!pVCpu->iem.s.fBypassHandlers)
1479 {
1480 VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
1481 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1482 { /* likely */ }
1483 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1484 {
1485 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1486 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1487 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1488 }
1489 else
1490 {
1491 Log((RT_SUCCESS(rcStrict)
1492 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1493 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1494 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1495 return rcStrict;
1496 }
1497 }
1498 else
1499 {
1500 rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
1501 if (RT_SUCCESS(rc))
1502 { /* likely */ }
1503 else
1504 {
1505 Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
1506 GCPtrPC, GCPhys, rc, cbToTryRead));
1507 return rc;
1508 }
1509 }
1510 pVCpu->iem.s.cbOpcode = cbToTryRead;
1511 }
1512#endif /* !IEM_WITH_CODE_TLB */
1513 return VINF_SUCCESS;
1514}
1515
1516
1517/**
1518 * Invalidates the IEM TLBs.
1519 *
1520 * This is called internally as well as by PGM when moving GC mappings.
1521 *
1522 * @returns
1523 * @param pVCpu The cross context virtual CPU structure of the calling
1524 * thread.
1525 * @param fVmm Set when PGM calls us with a remapping.
1526 */
1527VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPU pVCpu, bool fVmm)
1528{
1529#ifdef IEM_WITH_CODE_TLB
1530 pVCpu->iem.s.cbInstrBufTotal = 0;
1531 pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
1532 if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
1533 { /* very likely */ }
1534 else
1535 {
1536 pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
1537 unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
1538 while (i-- > 0)
1539 pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
1540 }
1541#endif
1542
1543#ifdef IEM_WITH_DATA_TLB
1544 pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
1545 if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
1546 { /* very likely */ }
1547 else
1548 {
1549 pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
1550 unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
1551 while (i-- > 0)
1552 pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
1553 }
1554#endif
1555 NOREF(pVCpu); NOREF(fVmm);
1556}
1557
1558
1559/**
1560 * Invalidates a page in the TLBs.
1561 *
1562 * @param pVCpu The cross context virtual CPU structure of the calling
1563 * thread.
1564 * @param GCPtr The address of the page to invalidate
1565 */
1566VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPU pVCpu, RTGCPTR GCPtr)
1567{
1568#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
1569 GCPtr = GCPtr >> X86_PAGE_SHIFT;
1570 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
1571 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries) == 256);
1572 uintptr_t idx = (uint8_t)GCPtr;
1573
1574# ifdef IEM_WITH_CODE_TLB
1575 if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision))
1576 {
1577 pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
1578 if (GCPtr == (pVCpu->iem.s.uInstrBufPc >> X86_PAGE_SHIFT))
1579 pVCpu->iem.s.cbInstrBufTotal = 0;
1580 }
1581# endif
1582
1583# ifdef IEM_WITH_DATA_TLB
1584 if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision))
1585 pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
1586# endif
1587#else
1588 NOREF(pVCpu); NOREF(GCPtr);
1589#endif
1590}
1591
1592
1593/**
1594 * Invalidates the host physical aspects of the IEM TLBs.
1595 *
1596 * This is called internally as well as by PGM when moving GC mappings.
1597 *
1598 * @param pVCpu The cross context virtual CPU structure of the calling
1599 * thread.
1600 */
1601VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPU pVCpu)
1602{
1603#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
1604 /* Note! This probably won't end up looking exactly like this, but it give an idea... */
1605
1606# ifdef IEM_WITH_CODE_TLB
1607 pVCpu->iem.s.cbInstrBufTotal = 0;
1608# endif
1609 uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
1610 if (uTlbPhysRev != 0)
1611 {
1612 pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
1613 pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
1614 }
1615 else
1616 {
1617 pVCpu->iem.s.CodeTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
1618 pVCpu->iem.s.DataTlb.uTlbPhysRev = IEMTLB_PHYS_REV_INCR;
1619
1620 unsigned i;
1621# ifdef IEM_WITH_CODE_TLB
1622 i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
1623 while (i-- > 0)
1624 {
1625 pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3 = NULL;
1626 pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
1627 }
1628# endif
1629# ifdef IEM_WITH_DATA_TLB
1630 i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
1631 while (i-- > 0)
1632 {
1633 pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3 = NULL;
1634 pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~(IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PHYS_REV);
1635 }
1636# endif
1637 }
1638#else
1639 NOREF(pVCpu);
1640#endif
1641}
1642
1643
1644/**
1645 * Invalidates the host physical aspects of the IEM TLBs.
1646 *
1647 * This is called internally as well as by PGM when moving GC mappings.
1648 *
1649 * @param pVM The cross context VM structure.
1650 *
1651 * @remarks Caller holds the PGM lock.
1652 */
1653VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVM pVM)
1654{
1655 RT_NOREF_PV(pVM);
1656}
1657
1658#ifdef IEM_WITH_CODE_TLB
1659
1660/**
1661 * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
1662 * failure and jumps.
1663 *
1664 * We end up here for a number of reasons:
1665 * - pbInstrBuf isn't yet initialized.
1666 * - Advancing beyond the buffer boundrary (e.g. cross page).
1667 * - Advancing beyond the CS segment limit.
1668 * - Fetching from non-mappable page (e.g. MMIO).
1669 *
1670 * @param pVCpu The cross context virtual CPU structure of the
1671 * calling thread.
1672 * @param pvDst Where to return the bytes.
1673 * @param cbDst Number of bytes to read.
1674 *
1675 * @todo Make cbDst = 0 a way of initializing pbInstrBuf?
1676 */
1677IEM_STATIC void iemOpcodeFetchBytesJmp(PVMCPU pVCpu, size_t cbDst, void *pvDst)
1678{
1679#ifdef IN_RING3
1680 for (;;)
1681 {
1682 Assert(cbDst <= 8);
1683 uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
1684
1685 /*
1686 * We might have a partial buffer match, deal with that first to make the
1687 * rest simpler. This is the first part of the cross page/buffer case.
1688 */
1689 if (pVCpu->iem.s.pbInstrBuf != NULL)
1690 {
1691 if (offBuf < pVCpu->iem.s.cbInstrBuf)
1692 {
1693 Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
1694 uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
1695 memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
1696
1697 cbDst -= cbCopy;
1698 pvDst = (uint8_t *)pvDst + cbCopy;
1699 offBuf += cbCopy;
1700 pVCpu->iem.s.offInstrNextByte += offBuf;
1701 }
1702 }
1703
1704 /*
1705 * Check segment limit, figuring how much we're allowed to access at this point.
1706 *
1707 * We will fault immediately if RIP is past the segment limit / in non-canonical
1708 * territory. If we do continue, there are one or more bytes to read before we
1709 * end up in trouble and we need to do that first before faulting.
1710 */
1711 RTGCPTR GCPtrFirst;
1712 uint32_t cbMaxRead;
1713 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1714 {
1715 GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
1716 if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
1717 { /* likely */ }
1718 else
1719 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
1720 cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
1721 }
1722 else
1723 {
1724 GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
1725 Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1726 if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
1727 { /* likely */ }
1728 else
1729 iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1730 cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
1731 if (cbMaxRead != 0)
1732 { /* likely */ }
1733 else
1734 {
1735 /* Overflowed because address is 0 and limit is max. */
1736 Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1737 cbMaxRead = X86_PAGE_SIZE;
1738 }
1739 GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
1740 uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
1741 if (cbMaxRead2 < cbMaxRead)
1742 cbMaxRead = cbMaxRead2;
1743 /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
1744 }
1745
1746 /*
1747 * Get the TLB entry for this piece of code.
1748 */
1749 uint64_t uTag = (GCPtrFirst >> X86_PAGE_SHIFT) | pVCpu->iem.s.CodeTlb.uTlbRevision;
1750 AssertCompile(RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries) == 256);
1751 PIEMTLBENTRY pTlbe = &pVCpu->iem.s.CodeTlb.aEntries[(uint8_t)uTag];
1752 if (pTlbe->uTag == uTag)
1753 {
1754 /* likely when executing lots of code, otherwise unlikely */
1755# ifdef VBOX_WITH_STATISTICS
1756 pVCpu->iem.s.CodeTlb.cTlbHits++;
1757# endif
1758 }
1759 else
1760 {
1761 pVCpu->iem.s.CodeTlb.cTlbMisses++;
1762# ifdef VBOX_WITH_RAW_MODE_NOT_R0
1763 if (PATMIsPatchGCAddr(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.GstCtx.eip))
1764 {
1765 pTlbe->uTag = uTag;
1766 pTlbe->fFlagsAndPhysRev = IEMTLBE_F_PATCH_CODE | IEMTLBE_F_PT_NO_WRITE | IEMTLBE_F_PT_NO_USER
1767 | IEMTLBE_F_PT_NO_WRITE | IEMTLBE_F_PT_NO_DIRTY | IEMTLBE_F_NO_MAPPINGR3;
1768 pTlbe->GCPhys = NIL_RTGCPHYS;
1769 pTlbe->pbMappingR3 = NULL;
1770 }
1771 else
1772# endif
1773 {
1774 RTGCPHYS GCPhys;
1775 uint64_t fFlags;
1776 int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &fFlags, &GCPhys);
1777 if (RT_FAILURE(rc))
1778 {
1779 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
1780 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, rc);
1781 }
1782
1783 AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
1784 pTlbe->uTag = uTag;
1785 pTlbe->fFlagsAndPhysRev = (~fFlags & (X86_PTE_US | X86_PTE_RW | X86_PTE_D)) | (fFlags >> X86_PTE_PAE_BIT_NX);
1786 pTlbe->GCPhys = GCPhys;
1787 pTlbe->pbMappingR3 = NULL;
1788 }
1789 }
1790
1791 /*
1792 * Check TLB page table level access flags.
1793 */
1794 if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
1795 {
1796 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
1797 {
1798 Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
1799 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1800 }
1801 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
1802 {
1803 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
1804 iemRaisePageFaultJmp(pVCpu, GCPtrFirst, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
1805 }
1806 }
1807
1808# ifdef VBOX_WITH_RAW_MODE_NOT_R0
1809 /*
1810 * Allow interpretation of patch manager code blocks since they can for
1811 * instance throw #PFs for perfectly good reasons.
1812 */
1813 if (!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PATCH_CODE))
1814 { /* no unlikely */ }
1815 else
1816 {
1817 /** @todo Could be optimized this a little in ring-3 if we liked. */
1818 size_t cbRead = 0;
1819 int rc = PATMReadPatchCode(pVCpu->CTX_SUFF(pVM), GCPtrFirst, pvDst, cbDst, &cbRead);
1820 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
1821 AssertStmt(cbRead == cbDst, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_IEM_IPE_1));
1822 return;
1823 }
1824# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
1825
1826 /*
1827 * Look up the physical page info if necessary.
1828 */
1829 if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1830 { /* not necessary */ }
1831 else
1832 {
1833 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE == IEMTLBE_F_PG_NO_WRITE);
1834 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ == IEMTLBE_F_PG_NO_READ);
1835 AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
1836 pTlbe->fFlagsAndPhysRev &= ~( IEMTLBE_F_PHYS_REV
1837 | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE);
1838 int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
1839 &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
1840 AssertRCStmt(rc, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), rc));
1841 }
1842
1843# if defined(IN_RING3) || (defined(IN_RING0) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE))
1844 /*
1845 * Try do a direct read using the pbMappingR3 pointer.
1846 */
1847 if ( (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
1848 == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1849 {
1850 uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
1851 pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
1852 if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
1853 {
1854 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(15, cbMaxRead);
1855 pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
1856 }
1857 else
1858 {
1859 uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
1860 Assert(cbInstr < cbMaxRead);
1861 pVCpu->iem.s.cbInstrBuf = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
1862 pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
1863 }
1864 if (cbDst <= cbMaxRead)
1865 {
1866 pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
1867 pVCpu->iem.s.uInstrBufPc = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
1868 pVCpu->iem.s.pbInstrBuf = pTlbe->pbMappingR3;
1869 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
1870 return;
1871 }
1872 pVCpu->iem.s.pbInstrBuf = NULL;
1873
1874 memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
1875 pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
1876 }
1877 else
1878# endif
1879#if 0
1880 /*
1881 * If there is no special read handling, so we can read a bit more and
1882 * put it in the prefetch buffer.
1883 */
1884 if ( cbDst < cbMaxRead
1885 && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
1886 {
1887 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
1888 &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
1889 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1890 { /* likely */ }
1891 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1892 {
1893 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1894 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1895 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1896 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICRC_VAL(rcStrict)));
1897 }
1898 else
1899 {
1900 Log((RT_SUCCESS(rcStrict)
1901 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1902 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1903 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
1904 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1905 }
1906 }
1907 /*
1908 * Special read handling, so only read exactly what's needed.
1909 * This is a highly unlikely scenario.
1910 */
1911 else
1912#endif
1913 {
1914 pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
1915 uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
1916 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
1917 pvDst, cbToRead, PGMACCESSORIGIN_IEM);
1918 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1919 { /* likely */ }
1920 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
1921 {
1922 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
1923 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1924 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
1925 AssertStmt(rcStrict == VINF_SUCCESS, longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict)));
1926 }
1927 else
1928 {
1929 Log((RT_SUCCESS(rcStrict)
1930 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
1931 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
1932 GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
1933 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
1934 }
1935 pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
1936 if (cbToRead == cbDst)
1937 return;
1938 }
1939
1940 /*
1941 * More to read, loop.
1942 */
1943 cbDst -= cbMaxRead;
1944 pvDst = (uint8_t *)pvDst + cbMaxRead;
1945 }
1946#else
1947 RT_NOREF(pvDst, cbDst);
1948 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VERR_INTERNAL_ERROR);
1949#endif
1950}
1951
1952#else
1953
1954/**
1955 * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
1956 * exception if it fails.
1957 *
1958 * @returns Strict VBox status code.
1959 * @param pVCpu The cross context virtual CPU structure of the
1960 * calling thread.
1961 * @param cbMin The minimum number of bytes relative offOpcode
1962 * that must be read.
1963 */
1964IEM_STATIC VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPU pVCpu, size_t cbMin)
1965{
1966 /*
1967 * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
1968 *
1969 * First translate CS:rIP to a physical address.
1970 */
1971 uint8_t cbLeft = pVCpu->iem.s.cbOpcode - pVCpu->iem.s.offOpcode; Assert(cbLeft < cbMin);
1972 uint32_t cbToTryRead;
1973 RTGCPTR GCPtrNext;
1974 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1975 {
1976 cbToTryRead = PAGE_SIZE;
1977 GCPtrNext = pVCpu->cpum.GstCtx.rip + pVCpu->iem.s.cbOpcode;
1978 if (!IEM_IS_CANONICAL(GCPtrNext))
1979 return iemRaiseGeneralProtectionFault0(pVCpu);
1980 }
1981 else
1982 {
1983 uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
1984 Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT);
1985 GCPtrNext32 += pVCpu->iem.s.cbOpcode;
1986 if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
1987 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1988 cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
1989 if (!cbToTryRead) /* overflowed */
1990 {
1991 Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
1992 cbToTryRead = UINT32_MAX;
1993 /** @todo check out wrapping around the code segment. */
1994 }
1995 if (cbToTryRead < cbMin - cbLeft)
1996 return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
1997 GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
1998 }
1999
2000 /* Only read up to the end of the page, and make sure we don't read more
2001 than the opcode buffer can hold. */
2002 uint32_t cbLeftOnPage = PAGE_SIZE - (GCPtrNext & PAGE_OFFSET_MASK);
2003 if (cbToTryRead > cbLeftOnPage)
2004 cbToTryRead = cbLeftOnPage;
2005 if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode)
2006 cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - pVCpu->iem.s.cbOpcode;
2007/** @todo r=bird: Convert assertion into undefined opcode exception? */
2008 Assert(cbToTryRead >= cbMin - cbLeft); /* ASSUMPTION based on iemInitDecoderAndPrefetchOpcodes. */
2009
2010# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2011 /* Allow interpretation of patch manager code blocks since they can for
2012 instance throw #PFs for perfectly good reasons. */
2013 if (pVCpu->iem.s.fInPatchCode)
2014 {
2015 size_t cbRead = 0;
2016 int rc = PATMReadPatchCode(pVCpu->CTX_SUFF(pVM), GCPtrNext, pVCpu->iem.s.abOpcode, cbToTryRead, &cbRead);
2017 AssertRCReturn(rc, rc);
2018 pVCpu->iem.s.cbOpcode = (uint8_t)cbRead; Assert(pVCpu->iem.s.cbOpcode == cbRead); Assert(cbRead > 0);
2019 return VINF_SUCCESS;
2020 }
2021# endif /* VBOX_WITH_RAW_MODE_NOT_R0 */
2022
2023 RTGCPHYS GCPhys;
2024 uint64_t fFlags;
2025 int rc = PGMGstGetPage(pVCpu, GCPtrNext, &fFlags, &GCPhys);
2026 if (RT_FAILURE(rc))
2027 {
2028 Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
2029 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, rc);
2030 }
2031 if (!(fFlags & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
2032 {
2033 Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
2034 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
2035 }
2036 if ((fFlags & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
2037 {
2038 Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
2039 return iemRaisePageFault(pVCpu, GCPtrNext, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
2040 }
2041 GCPhys |= GCPtrNext & PAGE_OFFSET_MASK;
2042 Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n", GCPtrNext, GCPhys, pVCpu->iem.s.cbOpcode));
2043 /** @todo Check reserved bits and such stuff. PGM is better at doing
2044 * that, so do it when implementing the guest virtual address
2045 * TLB... */
2046
2047 /*
2048 * Read the bytes at this address.
2049 *
2050 * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
2051 * and since PATM should only patch the start of an instruction there
2052 * should be no need to check again here.
2053 */
2054 if (!pVCpu->iem.s.fBypassHandlers)
2055 {
2056 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode],
2057 cbToTryRead, PGMACCESSORIGIN_IEM);
2058 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2059 { /* likely */ }
2060 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
2061 {
2062 Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n",
2063 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
2064 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
2065 }
2066 else
2067 {
2068 Log((RT_SUCCESS(rcStrict)
2069 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
2070 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
2071 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
2072 return rcStrict;
2073 }
2074 }
2075 else
2076 {
2077 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[pVCpu->iem.s.cbOpcode], GCPhys, cbToTryRead);
2078 if (RT_SUCCESS(rc))
2079 { /* likely */ }
2080 else
2081 {
2082 Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
2083 return rc;
2084 }
2085 }
2086 pVCpu->iem.s.cbOpcode += cbToTryRead;
2087 Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
2088
2089 return VINF_SUCCESS;
2090}
2091
2092#endif /* !IEM_WITH_CODE_TLB */
2093#ifndef IEM_WITH_SETJMP
2094
2095/**
2096 * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
2097 *
2098 * @returns Strict VBox status code.
2099 * @param pVCpu The cross context virtual CPU structure of the
2100 * calling thread.
2101 * @param pb Where to return the opcode byte.
2102 */
2103DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU8Slow(PVMCPU pVCpu, uint8_t *pb)
2104{
2105 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
2106 if (rcStrict == VINF_SUCCESS)
2107 {
2108 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2109 *pb = pVCpu->iem.s.abOpcode[offOpcode];
2110 pVCpu->iem.s.offOpcode = offOpcode + 1;
2111 }
2112 else
2113 *pb = 0;
2114 return rcStrict;
2115}
2116
2117
2118/**
2119 * Fetches the next opcode byte.
2120 *
2121 * @returns Strict VBox status code.
2122 * @param pVCpu The cross context virtual CPU structure of the
2123 * calling thread.
2124 * @param pu8 Where to return the opcode byte.
2125 */
2126DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPU pVCpu, uint8_t *pu8)
2127{
2128 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2129 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2130 {
2131 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2132 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
2133 return VINF_SUCCESS;
2134 }
2135 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
2136}
2137
2138#else /* IEM_WITH_SETJMP */
2139
2140/**
2141 * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
2142 *
2143 * @returns The opcode byte.
2144 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2145 */
2146DECL_NO_INLINE(IEM_STATIC, uint8_t) iemOpcodeGetNextU8SlowJmp(PVMCPU pVCpu)
2147{
2148# ifdef IEM_WITH_CODE_TLB
2149 uint8_t u8;
2150 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
2151 return u8;
2152# else
2153 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
2154 if (rcStrict == VINF_SUCCESS)
2155 return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
2156 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2157# endif
2158}
2159
2160
2161/**
2162 * Fetches the next opcode byte, longjmp on error.
2163 *
2164 * @returns The opcode byte.
2165 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2166 */
2167DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPU pVCpu)
2168{
2169# ifdef IEM_WITH_CODE_TLB
2170 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2171 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2172 if (RT_LIKELY( pbBuf != NULL
2173 && offBuf < pVCpu->iem.s.cbInstrBuf))
2174 {
2175 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
2176 return pbBuf[offBuf];
2177 }
2178# else
2179 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
2180 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2181 {
2182 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2183 return pVCpu->iem.s.abOpcode[offOpcode];
2184 }
2185# endif
2186 return iemOpcodeGetNextU8SlowJmp(pVCpu);
2187}
2188
2189#endif /* IEM_WITH_SETJMP */
2190
2191/**
2192 * Fetches the next opcode byte, returns automatically on failure.
2193 *
2194 * @param a_pu8 Where to return the opcode byte.
2195 * @remark Implicitly references pVCpu.
2196 */
2197#ifndef IEM_WITH_SETJMP
2198# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
2199 do \
2200 { \
2201 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
2202 if (rcStrict2 == VINF_SUCCESS) \
2203 { /* likely */ } \
2204 else \
2205 return rcStrict2; \
2206 } while (0)
2207#else
2208# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
2209#endif /* IEM_WITH_SETJMP */
2210
2211
2212#ifndef IEM_WITH_SETJMP
2213/**
2214 * Fetches the next signed byte from the opcode stream.
2215 *
2216 * @returns Strict VBox status code.
2217 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2218 * @param pi8 Where to return the signed byte.
2219 */
2220DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPU pVCpu, int8_t *pi8)
2221{
2222 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
2223}
2224#endif /* !IEM_WITH_SETJMP */
2225
2226
2227/**
2228 * Fetches the next signed byte from the opcode stream, returning automatically
2229 * on failure.
2230 *
2231 * @param a_pi8 Where to return the signed byte.
2232 * @remark Implicitly references pVCpu.
2233 */
2234#ifndef IEM_WITH_SETJMP
2235# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
2236 do \
2237 { \
2238 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
2239 if (rcStrict2 != VINF_SUCCESS) \
2240 return rcStrict2; \
2241 } while (0)
2242#else /* IEM_WITH_SETJMP */
2243# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2244
2245#endif /* IEM_WITH_SETJMP */
2246
2247#ifndef IEM_WITH_SETJMP
2248
2249/**
2250 * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
2251 *
2252 * @returns Strict VBox status code.
2253 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2254 * @param pu16 Where to return the opcode dword.
2255 */
2256DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU16Slow(PVMCPU pVCpu, uint16_t *pu16)
2257{
2258 uint8_t u8;
2259 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2260 if (rcStrict == VINF_SUCCESS)
2261 *pu16 = (int8_t)u8;
2262 return rcStrict;
2263}
2264
2265
2266/**
2267 * Fetches the next signed byte from the opcode stream, extending it to
2268 * unsigned 16-bit.
2269 *
2270 * @returns Strict VBox status code.
2271 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2272 * @param pu16 Where to return the unsigned word.
2273 */
2274DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPU pVCpu, uint16_t *pu16)
2275{
2276 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2277 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2278 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
2279
2280 *pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2281 pVCpu->iem.s.offOpcode = offOpcode + 1;
2282 return VINF_SUCCESS;
2283}
2284
2285#endif /* !IEM_WITH_SETJMP */
2286
2287/**
2288 * Fetches the next signed byte from the opcode stream and sign-extending it to
2289 * a word, returning automatically on failure.
2290 *
2291 * @param a_pu16 Where to return the word.
2292 * @remark Implicitly references pVCpu.
2293 */
2294#ifndef IEM_WITH_SETJMP
2295# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
2296 do \
2297 { \
2298 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
2299 if (rcStrict2 != VINF_SUCCESS) \
2300 return rcStrict2; \
2301 } while (0)
2302#else
2303# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2304#endif
2305
2306#ifndef IEM_WITH_SETJMP
2307
2308/**
2309 * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
2310 *
2311 * @returns Strict VBox status code.
2312 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2313 * @param pu32 Where to return the opcode dword.
2314 */
2315DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU32Slow(PVMCPU pVCpu, uint32_t *pu32)
2316{
2317 uint8_t u8;
2318 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2319 if (rcStrict == VINF_SUCCESS)
2320 *pu32 = (int8_t)u8;
2321 return rcStrict;
2322}
2323
2324
2325/**
2326 * Fetches the next signed byte from the opcode stream, extending it to
2327 * unsigned 32-bit.
2328 *
2329 * @returns Strict VBox status code.
2330 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2331 * @param pu32 Where to return the unsigned dword.
2332 */
2333DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPU pVCpu, uint32_t *pu32)
2334{
2335 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2336 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2337 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
2338
2339 *pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2340 pVCpu->iem.s.offOpcode = offOpcode + 1;
2341 return VINF_SUCCESS;
2342}
2343
2344#endif /* !IEM_WITH_SETJMP */
2345
2346/**
2347 * Fetches the next signed byte from the opcode stream and sign-extending it to
2348 * a word, returning automatically on failure.
2349 *
2350 * @param a_pu32 Where to return the word.
2351 * @remark Implicitly references pVCpu.
2352 */
2353#ifndef IEM_WITH_SETJMP
2354#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
2355 do \
2356 { \
2357 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
2358 if (rcStrict2 != VINF_SUCCESS) \
2359 return rcStrict2; \
2360 } while (0)
2361#else
2362# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2363#endif
2364
2365#ifndef IEM_WITH_SETJMP
2366
2367/**
2368 * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
2369 *
2370 * @returns Strict VBox status code.
2371 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2372 * @param pu64 Where to return the opcode qword.
2373 */
2374DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS8SxU64Slow(PVMCPU pVCpu, uint64_t *pu64)
2375{
2376 uint8_t u8;
2377 VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
2378 if (rcStrict == VINF_SUCCESS)
2379 *pu64 = (int8_t)u8;
2380 return rcStrict;
2381}
2382
2383
2384/**
2385 * Fetches the next signed byte from the opcode stream, extending it to
2386 * unsigned 64-bit.
2387 *
2388 * @returns Strict VBox status code.
2389 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2390 * @param pu64 Where to return the unsigned qword.
2391 */
2392DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPU pVCpu, uint64_t *pu64)
2393{
2394 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2395 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
2396 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
2397
2398 *pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
2399 pVCpu->iem.s.offOpcode = offOpcode + 1;
2400 return VINF_SUCCESS;
2401}
2402
2403#endif /* !IEM_WITH_SETJMP */
2404
2405
2406/**
2407 * Fetches the next signed byte from the opcode stream and sign-extending it to
2408 * a word, returning automatically on failure.
2409 *
2410 * @param a_pu64 Where to return the word.
2411 * @remark Implicitly references pVCpu.
2412 */
2413#ifndef IEM_WITH_SETJMP
2414# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
2415 do \
2416 { \
2417 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
2418 if (rcStrict2 != VINF_SUCCESS) \
2419 return rcStrict2; \
2420 } while (0)
2421#else
2422# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
2423#endif
2424
2425
2426#ifndef IEM_WITH_SETJMP
2427/**
2428 * Fetches the next opcode byte.
2429 *
2430 * @returns Strict VBox status code.
2431 * @param pVCpu The cross context virtual CPU structure of the
2432 * calling thread.
2433 * @param pu8 Where to return the opcode byte.
2434 */
2435DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPU pVCpu, uint8_t *pu8)
2436{
2437 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2438 pVCpu->iem.s.offModRm = offOpcode;
2439 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2440 {
2441 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2442 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
2443 return VINF_SUCCESS;
2444 }
2445 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
2446}
2447#else /* IEM_WITH_SETJMP */
2448/**
2449 * Fetches the next opcode byte, longjmp on error.
2450 *
2451 * @returns The opcode byte.
2452 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2453 */
2454DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPU pVCpu)
2455{
2456# ifdef IEM_WITH_CODE_TLB
2457 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2458 pVCpu->iem.s.offModRm = offBuf;
2459 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2460 if (RT_LIKELY( pbBuf != NULL
2461 && offBuf < pVCpu->iem.s.cbInstrBuf))
2462 {
2463 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
2464 return pbBuf[offBuf];
2465 }
2466# else
2467 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
2468 pVCpu->iem.s.offModRm = offOpcode;
2469 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
2470 {
2471 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
2472 return pVCpu->iem.s.abOpcode[offOpcode];
2473 }
2474# endif
2475 return iemOpcodeGetNextU8SlowJmp(pVCpu);
2476}
2477#endif /* IEM_WITH_SETJMP */
2478
2479/**
2480 * Fetches the next opcode byte, which is a ModR/M byte, returns automatically
2481 * on failure.
2482 *
2483 * Will note down the position of the ModR/M byte for VT-x exits.
2484 *
2485 * @param a_pbRm Where to return the RM opcode byte.
2486 * @remark Implicitly references pVCpu.
2487 */
2488#ifndef IEM_WITH_SETJMP
2489# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
2490 do \
2491 { \
2492 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
2493 if (rcStrict2 == VINF_SUCCESS) \
2494 { /* likely */ } \
2495 else \
2496 return rcStrict2; \
2497 } while (0)
2498#else
2499# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
2500#endif /* IEM_WITH_SETJMP */
2501
2502
2503#ifndef IEM_WITH_SETJMP
2504
2505/**
2506 * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
2507 *
2508 * @returns Strict VBox status code.
2509 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2510 * @param pu16 Where to return the opcode word.
2511 */
2512DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16Slow(PVMCPU pVCpu, uint16_t *pu16)
2513{
2514 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2515 if (rcStrict == VINF_SUCCESS)
2516 {
2517 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2518# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2519 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2520# else
2521 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2522# endif
2523 pVCpu->iem.s.offOpcode = offOpcode + 2;
2524 }
2525 else
2526 *pu16 = 0;
2527 return rcStrict;
2528}
2529
2530
2531/**
2532 * Fetches the next opcode word.
2533 *
2534 * @returns Strict VBox status code.
2535 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2536 * @param pu16 Where to return the opcode word.
2537 */
2538DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPU pVCpu, uint16_t *pu16)
2539{
2540 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2541 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
2542 {
2543 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
2544# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2545 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2546# else
2547 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2548# endif
2549 return VINF_SUCCESS;
2550 }
2551 return iemOpcodeGetNextU16Slow(pVCpu, pu16);
2552}
2553
2554#else /* IEM_WITH_SETJMP */
2555
2556/**
2557 * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
2558 *
2559 * @returns The opcode word.
2560 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2561 */
2562DECL_NO_INLINE(IEM_STATIC, uint16_t) iemOpcodeGetNextU16SlowJmp(PVMCPU pVCpu)
2563{
2564# ifdef IEM_WITH_CODE_TLB
2565 uint16_t u16;
2566 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
2567 return u16;
2568# else
2569 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2570 if (rcStrict == VINF_SUCCESS)
2571 {
2572 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2573 pVCpu->iem.s.offOpcode += 2;
2574# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2575 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2576# else
2577 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2578# endif
2579 }
2580 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2581# endif
2582}
2583
2584
2585/**
2586 * Fetches the next opcode word, longjmp on error.
2587 *
2588 * @returns The opcode word.
2589 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2590 */
2591DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPU pVCpu)
2592{
2593# ifdef IEM_WITH_CODE_TLB
2594 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2595 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2596 if (RT_LIKELY( pbBuf != NULL
2597 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
2598 {
2599 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
2600# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2601 return *(uint16_t const *)&pbBuf[offBuf];
2602# else
2603 return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
2604# endif
2605 }
2606# else
2607 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2608 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
2609 {
2610 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
2611# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2612 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2613# else
2614 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2615# endif
2616 }
2617# endif
2618 return iemOpcodeGetNextU16SlowJmp(pVCpu);
2619}
2620
2621#endif /* IEM_WITH_SETJMP */
2622
2623
2624/**
2625 * Fetches the next opcode word, returns automatically on failure.
2626 *
2627 * @param a_pu16 Where to return the opcode word.
2628 * @remark Implicitly references pVCpu.
2629 */
2630#ifndef IEM_WITH_SETJMP
2631# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
2632 do \
2633 { \
2634 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
2635 if (rcStrict2 != VINF_SUCCESS) \
2636 return rcStrict2; \
2637 } while (0)
2638#else
2639# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
2640#endif
2641
2642#ifndef IEM_WITH_SETJMP
2643
2644/**
2645 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
2646 *
2647 * @returns Strict VBox status code.
2648 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2649 * @param pu32 Where to return the opcode double word.
2650 */
2651DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32Slow(PVMCPU pVCpu, uint32_t *pu32)
2652{
2653 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2654 if (rcStrict == VINF_SUCCESS)
2655 {
2656 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2657 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2658 pVCpu->iem.s.offOpcode = offOpcode + 2;
2659 }
2660 else
2661 *pu32 = 0;
2662 return rcStrict;
2663}
2664
2665
2666/**
2667 * Fetches the next opcode word, zero extending it to a double word.
2668 *
2669 * @returns Strict VBox status code.
2670 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2671 * @param pu32 Where to return the opcode double word.
2672 */
2673DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPU pVCpu, uint32_t *pu32)
2674{
2675 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2676 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
2677 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
2678
2679 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2680 pVCpu->iem.s.offOpcode = offOpcode + 2;
2681 return VINF_SUCCESS;
2682}
2683
2684#endif /* !IEM_WITH_SETJMP */
2685
2686
2687/**
2688 * Fetches the next opcode word and zero extends it to a double word, returns
2689 * automatically on failure.
2690 *
2691 * @param a_pu32 Where to return the opcode double word.
2692 * @remark Implicitly references pVCpu.
2693 */
2694#ifndef IEM_WITH_SETJMP
2695# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
2696 do \
2697 { \
2698 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
2699 if (rcStrict2 != VINF_SUCCESS) \
2700 return rcStrict2; \
2701 } while (0)
2702#else
2703# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
2704#endif
2705
2706#ifndef IEM_WITH_SETJMP
2707
2708/**
2709 * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
2710 *
2711 * @returns Strict VBox status code.
2712 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2713 * @param pu64 Where to return the opcode quad word.
2714 */
2715DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64Slow(PVMCPU pVCpu, uint64_t *pu64)
2716{
2717 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
2718 if (rcStrict == VINF_SUCCESS)
2719 {
2720 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2721 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2722 pVCpu->iem.s.offOpcode = offOpcode + 2;
2723 }
2724 else
2725 *pu64 = 0;
2726 return rcStrict;
2727}
2728
2729
2730/**
2731 * Fetches the next opcode word, zero extending it to a quad word.
2732 *
2733 * @returns Strict VBox status code.
2734 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2735 * @param pu64 Where to return the opcode quad word.
2736 */
2737DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPU pVCpu, uint64_t *pu64)
2738{
2739 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2740 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
2741 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
2742
2743 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
2744 pVCpu->iem.s.offOpcode = offOpcode + 2;
2745 return VINF_SUCCESS;
2746}
2747
2748#endif /* !IEM_WITH_SETJMP */
2749
2750/**
2751 * Fetches the next opcode word and zero extends it to a quad word, returns
2752 * automatically on failure.
2753 *
2754 * @param a_pu64 Where to return the opcode quad word.
2755 * @remark Implicitly references pVCpu.
2756 */
2757#ifndef IEM_WITH_SETJMP
2758# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
2759 do \
2760 { \
2761 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
2762 if (rcStrict2 != VINF_SUCCESS) \
2763 return rcStrict2; \
2764 } while (0)
2765#else
2766# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
2767#endif
2768
2769
2770#ifndef IEM_WITH_SETJMP
2771/**
2772 * Fetches the next signed word from the opcode stream.
2773 *
2774 * @returns Strict VBox status code.
2775 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2776 * @param pi16 Where to return the signed word.
2777 */
2778DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPU pVCpu, int16_t *pi16)
2779{
2780 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
2781}
2782#endif /* !IEM_WITH_SETJMP */
2783
2784
2785/**
2786 * Fetches the next signed word from the opcode stream, returning automatically
2787 * on failure.
2788 *
2789 * @param a_pi16 Where to return the signed word.
2790 * @remark Implicitly references pVCpu.
2791 */
2792#ifndef IEM_WITH_SETJMP
2793# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
2794 do \
2795 { \
2796 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
2797 if (rcStrict2 != VINF_SUCCESS) \
2798 return rcStrict2; \
2799 } while (0)
2800#else
2801# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
2802#endif
2803
2804#ifndef IEM_WITH_SETJMP
2805
2806/**
2807 * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
2808 *
2809 * @returns Strict VBox status code.
2810 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2811 * @param pu32 Where to return the opcode dword.
2812 */
2813DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32Slow(PVMCPU pVCpu, uint32_t *pu32)
2814{
2815 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2816 if (rcStrict == VINF_SUCCESS)
2817 {
2818 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2819# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2820 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2821# else
2822 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2823 pVCpu->iem.s.abOpcode[offOpcode + 1],
2824 pVCpu->iem.s.abOpcode[offOpcode + 2],
2825 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2826# endif
2827 pVCpu->iem.s.offOpcode = offOpcode + 4;
2828 }
2829 else
2830 *pu32 = 0;
2831 return rcStrict;
2832}
2833
2834
2835/**
2836 * Fetches the next opcode dword.
2837 *
2838 * @returns Strict VBox status code.
2839 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2840 * @param pu32 Where to return the opcode double word.
2841 */
2842DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPU pVCpu, uint32_t *pu32)
2843{
2844 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2845 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
2846 {
2847 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
2848# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2849 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2850# else
2851 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2852 pVCpu->iem.s.abOpcode[offOpcode + 1],
2853 pVCpu->iem.s.abOpcode[offOpcode + 2],
2854 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2855# endif
2856 return VINF_SUCCESS;
2857 }
2858 return iemOpcodeGetNextU32Slow(pVCpu, pu32);
2859}
2860
2861#else /* !IEM_WITH_SETJMP */
2862
2863/**
2864 * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
2865 *
2866 * @returns The opcode dword.
2867 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2868 */
2869DECL_NO_INLINE(IEM_STATIC, uint32_t) iemOpcodeGetNextU32SlowJmp(PVMCPU pVCpu)
2870{
2871# ifdef IEM_WITH_CODE_TLB
2872 uint32_t u32;
2873 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
2874 return u32;
2875# else
2876 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2877 if (rcStrict == VINF_SUCCESS)
2878 {
2879 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2880 pVCpu->iem.s.offOpcode = offOpcode + 4;
2881# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2882 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2883# else
2884 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2885 pVCpu->iem.s.abOpcode[offOpcode + 1],
2886 pVCpu->iem.s.abOpcode[offOpcode + 2],
2887 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2888# endif
2889 }
2890 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
2891# endif
2892}
2893
2894
2895/**
2896 * Fetches the next opcode dword, longjmp on error.
2897 *
2898 * @returns The opcode dword.
2899 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2900 */
2901DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPU pVCpu)
2902{
2903# ifdef IEM_WITH_CODE_TLB
2904 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
2905 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
2906 if (RT_LIKELY( pbBuf != NULL
2907 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
2908 {
2909 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
2910# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2911 return *(uint32_t const *)&pbBuf[offBuf];
2912# else
2913 return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
2914 pbBuf[offBuf + 1],
2915 pbBuf[offBuf + 2],
2916 pbBuf[offBuf + 3]);
2917# endif
2918 }
2919# else
2920 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
2921 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
2922 {
2923 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
2924# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
2925 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
2926# else
2927 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2928 pVCpu->iem.s.abOpcode[offOpcode + 1],
2929 pVCpu->iem.s.abOpcode[offOpcode + 2],
2930 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2931# endif
2932 }
2933# endif
2934 return iemOpcodeGetNextU32SlowJmp(pVCpu);
2935}
2936
2937#endif /* !IEM_WITH_SETJMP */
2938
2939
2940/**
2941 * Fetches the next opcode dword, returns automatically on failure.
2942 *
2943 * @param a_pu32 Where to return the opcode dword.
2944 * @remark Implicitly references pVCpu.
2945 */
2946#ifndef IEM_WITH_SETJMP
2947# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
2948 do \
2949 { \
2950 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
2951 if (rcStrict2 != VINF_SUCCESS) \
2952 return rcStrict2; \
2953 } while (0)
2954#else
2955# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
2956#endif
2957
2958#ifndef IEM_WITH_SETJMP
2959
2960/**
2961 * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
2962 *
2963 * @returns Strict VBox status code.
2964 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2965 * @param pu64 Where to return the opcode dword.
2966 */
2967DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64Slow(PVMCPU pVCpu, uint64_t *pu64)
2968{
2969 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
2970 if (rcStrict == VINF_SUCCESS)
2971 {
2972 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
2973 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2974 pVCpu->iem.s.abOpcode[offOpcode + 1],
2975 pVCpu->iem.s.abOpcode[offOpcode + 2],
2976 pVCpu->iem.s.abOpcode[offOpcode + 3]);
2977 pVCpu->iem.s.offOpcode = offOpcode + 4;
2978 }
2979 else
2980 *pu64 = 0;
2981 return rcStrict;
2982}
2983
2984
2985/**
2986 * Fetches the next opcode dword, zero extending it to a quad word.
2987 *
2988 * @returns Strict VBox status code.
2989 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2990 * @param pu64 Where to return the opcode quad word.
2991 */
2992DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPU pVCpu, uint64_t *pu64)
2993{
2994 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
2995 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
2996 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
2997
2998 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
2999 pVCpu->iem.s.abOpcode[offOpcode + 1],
3000 pVCpu->iem.s.abOpcode[offOpcode + 2],
3001 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3002 pVCpu->iem.s.offOpcode = offOpcode + 4;
3003 return VINF_SUCCESS;
3004}
3005
3006#endif /* !IEM_WITH_SETJMP */
3007
3008
3009/**
3010 * Fetches the next opcode dword and zero extends it to a quad word, returns
3011 * automatically on failure.
3012 *
3013 * @param a_pu64 Where to return the opcode quad word.
3014 * @remark Implicitly references pVCpu.
3015 */
3016#ifndef IEM_WITH_SETJMP
3017# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
3018 do \
3019 { \
3020 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
3021 if (rcStrict2 != VINF_SUCCESS) \
3022 return rcStrict2; \
3023 } while (0)
3024#else
3025# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
3026#endif
3027
3028
3029#ifndef IEM_WITH_SETJMP
3030/**
3031 * Fetches the next signed double word from the opcode stream.
3032 *
3033 * @returns Strict VBox status code.
3034 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3035 * @param pi32 Where to return the signed double word.
3036 */
3037DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPU pVCpu, int32_t *pi32)
3038{
3039 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
3040}
3041#endif
3042
3043/**
3044 * Fetches the next signed double word from the opcode stream, returning
3045 * automatically on failure.
3046 *
3047 * @param a_pi32 Where to return the signed double word.
3048 * @remark Implicitly references pVCpu.
3049 */
3050#ifndef IEM_WITH_SETJMP
3051# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
3052 do \
3053 { \
3054 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
3055 if (rcStrict2 != VINF_SUCCESS) \
3056 return rcStrict2; \
3057 } while (0)
3058#else
3059# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
3060#endif
3061
3062#ifndef IEM_WITH_SETJMP
3063
3064/**
3065 * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
3066 *
3067 * @returns Strict VBox status code.
3068 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3069 * @param pu64 Where to return the opcode qword.
3070 */
3071DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextS32SxU64Slow(PVMCPU pVCpu, uint64_t *pu64)
3072{
3073 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
3074 if (rcStrict == VINF_SUCCESS)
3075 {
3076 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3077 *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3078 pVCpu->iem.s.abOpcode[offOpcode + 1],
3079 pVCpu->iem.s.abOpcode[offOpcode + 2],
3080 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3081 pVCpu->iem.s.offOpcode = offOpcode + 4;
3082 }
3083 else
3084 *pu64 = 0;
3085 return rcStrict;
3086}
3087
3088
3089/**
3090 * Fetches the next opcode dword, sign extending it into a quad word.
3091 *
3092 * @returns Strict VBox status code.
3093 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3094 * @param pu64 Where to return the opcode quad word.
3095 */
3096DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPU pVCpu, uint64_t *pu64)
3097{
3098 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
3099 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
3100 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
3101
3102 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3103 pVCpu->iem.s.abOpcode[offOpcode + 1],
3104 pVCpu->iem.s.abOpcode[offOpcode + 2],
3105 pVCpu->iem.s.abOpcode[offOpcode + 3]);
3106 *pu64 = i32;
3107 pVCpu->iem.s.offOpcode = offOpcode + 4;
3108 return VINF_SUCCESS;
3109}
3110
3111#endif /* !IEM_WITH_SETJMP */
3112
3113
3114/**
3115 * Fetches the next opcode double word and sign extends it to a quad word,
3116 * returns automatically on failure.
3117 *
3118 * @param a_pu64 Where to return the opcode quad word.
3119 * @remark Implicitly references pVCpu.
3120 */
3121#ifndef IEM_WITH_SETJMP
3122# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
3123 do \
3124 { \
3125 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
3126 if (rcStrict2 != VINF_SUCCESS) \
3127 return rcStrict2; \
3128 } while (0)
3129#else
3130# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
3131#endif
3132
3133#ifndef IEM_WITH_SETJMP
3134
3135/**
3136 * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
3137 *
3138 * @returns Strict VBox status code.
3139 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3140 * @param pu64 Where to return the opcode qword.
3141 */
3142DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemOpcodeGetNextU64Slow(PVMCPU pVCpu, uint64_t *pu64)
3143{
3144 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
3145 if (rcStrict == VINF_SUCCESS)
3146 {
3147 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3148# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3149 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3150# else
3151 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3152 pVCpu->iem.s.abOpcode[offOpcode + 1],
3153 pVCpu->iem.s.abOpcode[offOpcode + 2],
3154 pVCpu->iem.s.abOpcode[offOpcode + 3],
3155 pVCpu->iem.s.abOpcode[offOpcode + 4],
3156 pVCpu->iem.s.abOpcode[offOpcode + 5],
3157 pVCpu->iem.s.abOpcode[offOpcode + 6],
3158 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3159# endif
3160 pVCpu->iem.s.offOpcode = offOpcode + 8;
3161 }
3162 else
3163 *pu64 = 0;
3164 return rcStrict;
3165}
3166
3167
3168/**
3169 * Fetches the next opcode qword.
3170 *
3171 * @returns Strict VBox status code.
3172 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3173 * @param pu64 Where to return the opcode qword.
3174 */
3175DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPU pVCpu, uint64_t *pu64)
3176{
3177 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
3178 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
3179 {
3180# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3181 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3182# else
3183 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3184 pVCpu->iem.s.abOpcode[offOpcode + 1],
3185 pVCpu->iem.s.abOpcode[offOpcode + 2],
3186 pVCpu->iem.s.abOpcode[offOpcode + 3],
3187 pVCpu->iem.s.abOpcode[offOpcode + 4],
3188 pVCpu->iem.s.abOpcode[offOpcode + 5],
3189 pVCpu->iem.s.abOpcode[offOpcode + 6],
3190 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3191# endif
3192 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
3193 return VINF_SUCCESS;
3194 }
3195 return iemOpcodeGetNextU64Slow(pVCpu, pu64);
3196}
3197
3198#else /* IEM_WITH_SETJMP */
3199
3200/**
3201 * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
3202 *
3203 * @returns The opcode qword.
3204 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3205 */
3206DECL_NO_INLINE(IEM_STATIC, uint64_t) iemOpcodeGetNextU64SlowJmp(PVMCPU pVCpu)
3207{
3208# ifdef IEM_WITH_CODE_TLB
3209 uint64_t u64;
3210 iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
3211 return u64;
3212# else
3213 VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
3214 if (rcStrict == VINF_SUCCESS)
3215 {
3216 uint8_t offOpcode = pVCpu->iem.s.offOpcode;
3217 pVCpu->iem.s.offOpcode = offOpcode + 8;
3218# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3219 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3220# else
3221 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3222 pVCpu->iem.s.abOpcode[offOpcode + 1],
3223 pVCpu->iem.s.abOpcode[offOpcode + 2],
3224 pVCpu->iem.s.abOpcode[offOpcode + 3],
3225 pVCpu->iem.s.abOpcode[offOpcode + 4],
3226 pVCpu->iem.s.abOpcode[offOpcode + 5],
3227 pVCpu->iem.s.abOpcode[offOpcode + 6],
3228 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3229# endif
3230 }
3231 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
3232# endif
3233}
3234
3235
3236/**
3237 * Fetches the next opcode qword, longjmp on error.
3238 *
3239 * @returns The opcode qword.
3240 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3241 */
3242DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPU pVCpu)
3243{
3244# ifdef IEM_WITH_CODE_TLB
3245 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
3246 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
3247 if (RT_LIKELY( pbBuf != NULL
3248 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
3249 {
3250 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
3251# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3252 return *(uint64_t const *)&pbBuf[offBuf];
3253# else
3254 return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
3255 pbBuf[offBuf + 1],
3256 pbBuf[offBuf + 2],
3257 pbBuf[offBuf + 3],
3258 pbBuf[offBuf + 4],
3259 pbBuf[offBuf + 5],
3260 pbBuf[offBuf + 6],
3261 pbBuf[offBuf + 7]);
3262# endif
3263 }
3264# else
3265 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
3266 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
3267 {
3268 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
3269# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
3270 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
3271# else
3272 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
3273 pVCpu->iem.s.abOpcode[offOpcode + 1],
3274 pVCpu->iem.s.abOpcode[offOpcode + 2],
3275 pVCpu->iem.s.abOpcode[offOpcode + 3],
3276 pVCpu->iem.s.abOpcode[offOpcode + 4],
3277 pVCpu->iem.s.abOpcode[offOpcode + 5],
3278 pVCpu->iem.s.abOpcode[offOpcode + 6],
3279 pVCpu->iem.s.abOpcode[offOpcode + 7]);
3280# endif
3281 }
3282# endif
3283 return iemOpcodeGetNextU64SlowJmp(pVCpu);
3284}
3285
3286#endif /* IEM_WITH_SETJMP */
3287
3288/**
3289 * Fetches the next opcode quad word, returns automatically on failure.
3290 *
3291 * @param a_pu64 Where to return the opcode quad word.
3292 * @remark Implicitly references pVCpu.
3293 */
3294#ifndef IEM_WITH_SETJMP
3295# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
3296 do \
3297 { \
3298 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
3299 if (rcStrict2 != VINF_SUCCESS) \
3300 return rcStrict2; \
3301 } while (0)
3302#else
3303# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
3304#endif
3305
3306
3307/** @name Misc Worker Functions.
3308 * @{
3309 */
3310
3311/**
3312 * Gets the exception class for the specified exception vector.
3313 *
3314 * @returns The class of the specified exception.
3315 * @param uVector The exception vector.
3316 */
3317IEM_STATIC IEMXCPTCLASS iemGetXcptClass(uint8_t uVector)
3318{
3319 Assert(uVector <= X86_XCPT_LAST);
3320 switch (uVector)
3321 {
3322 case X86_XCPT_DE:
3323 case X86_XCPT_TS:
3324 case X86_XCPT_NP:
3325 case X86_XCPT_SS:
3326 case X86_XCPT_GP:
3327 case X86_XCPT_SX: /* AMD only */
3328 return IEMXCPTCLASS_CONTRIBUTORY;
3329
3330 case X86_XCPT_PF:
3331 case X86_XCPT_VE: /* Intel only */
3332 return IEMXCPTCLASS_PAGE_FAULT;
3333
3334 case X86_XCPT_DF:
3335 return IEMXCPTCLASS_DOUBLE_FAULT;
3336 }
3337 return IEMXCPTCLASS_BENIGN;
3338}
3339
3340
3341/**
3342 * Evaluates how to handle an exception caused during delivery of another event
3343 * (exception / interrupt).
3344 *
3345 * @returns How to handle the recursive exception.
3346 * @param pVCpu The cross context virtual CPU structure of the
3347 * calling thread.
3348 * @param fPrevFlags The flags of the previous event.
3349 * @param uPrevVector The vector of the previous event.
3350 * @param fCurFlags The flags of the current exception.
3351 * @param uCurVector The vector of the current exception.
3352 * @param pfXcptRaiseInfo Where to store additional information about the
3353 * exception condition. Optional.
3354 */
3355VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPU pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
3356 uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
3357{
3358 /*
3359 * Only CPU exceptions can be raised while delivering other events, software interrupt
3360 * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
3361 */
3362 AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
3363 Assert(pVCpu); RT_NOREF(pVCpu);
3364 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
3365
3366 IEMXCPTRAISE enmRaise = IEMXCPTRAISE_CURRENT_XCPT;
3367 IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
3368 if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3369 {
3370 IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
3371 if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
3372 {
3373 IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
3374 if ( enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
3375 && ( enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
3376 || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
3377 {
3378 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
3379 fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
3380 : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
3381 Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
3382 uCurVector, pVCpu->cpum.GstCtx.cr2));
3383 }
3384 else if ( enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
3385 && enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY)
3386 {
3387 enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
3388 Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
3389 }
3390 else if ( enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
3391 && ( enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
3392 || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
3393 {
3394 enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
3395 Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
3396 }
3397 }
3398 else
3399 {
3400 if (uPrevVector == X86_XCPT_NMI)
3401 {
3402 fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
3403 if (uCurVector == X86_XCPT_PF)
3404 {
3405 fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF;
3406 Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
3407 }
3408 }
3409 else if ( uPrevVector == X86_XCPT_AC
3410 && uCurVector == X86_XCPT_AC)
3411 {
3412 enmRaise = IEMXCPTRAISE_CPU_HANG;
3413 fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
3414 Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
3415 }
3416 }
3417 }
3418 else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
3419 {
3420 fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
3421 if (uCurVector == X86_XCPT_PF)
3422 fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF;
3423 }
3424 else
3425 {
3426 Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
3427 fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
3428 }
3429
3430 if (pfXcptRaiseInfo)
3431 *pfXcptRaiseInfo = fRaiseInfo;
3432 return enmRaise;
3433}
3434
3435
3436/**
3437 * Enters the CPU shutdown state initiated by a triple fault or other
3438 * unrecoverable conditions.
3439 *
3440 * @returns Strict VBox status code.
3441 * @param pVCpu The cross context virtual CPU structure of the
3442 * calling thread.
3443 */
3444IEM_STATIC VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPU pVCpu)
3445{
3446 if (IEM_IS_SVM_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
3447 {
3448 Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
3449 IEM_RETURN_SVM_VMEXIT(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
3450 }
3451
3452 RT_NOREF(pVCpu);
3453 return VINF_EM_TRIPLE_FAULT;
3454}
3455
3456
3457/**
3458 * Validates a new SS segment.
3459 *
3460 * @returns VBox strict status code.
3461 * @param pVCpu The cross context virtual CPU structure of the
3462 * calling thread.
3463 * @param NewSS The new SS selctor.
3464 * @param uCpl The CPL to load the stack for.
3465 * @param pDesc Where to return the descriptor.
3466 */
3467IEM_STATIC VBOXSTRICTRC iemMiscValidateNewSS(PVMCPU pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc)
3468{
3469 /* Null selectors are not allowed (we're not called for dispatching
3470 interrupts with SS=0 in long mode). */
3471 if (!(NewSS & X86_SEL_MASK_OFF_RPL))
3472 {
3473 Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
3474 return iemRaiseTaskSwitchFault0(pVCpu);
3475 }
3476
3477 /** @todo testcase: check that the TSS.ssX RPL is checked. Also check when. */
3478 if ((NewSS & X86_SEL_RPL) != uCpl)
3479 {
3480 Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
3481 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3482 }
3483
3484 /*
3485 * Read the descriptor.
3486 */
3487 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
3488 if (rcStrict != VINF_SUCCESS)
3489 return rcStrict;
3490
3491 /*
3492 * Perform the descriptor validation documented for LSS, POP SS and MOV SS.
3493 */
3494 if (!pDesc->Legacy.Gen.u1DescType)
3495 {
3496 Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
3497 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3498 }
3499
3500 if ( (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
3501 || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
3502 {
3503 Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
3504 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3505 }
3506 if (pDesc->Legacy.Gen.u2Dpl != uCpl)
3507 {
3508 Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
3509 return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
3510 }
3511
3512 /* Is it there? */
3513 /** @todo testcase: Is this checked before the canonical / limit check below? */
3514 if (!pDesc->Legacy.Gen.u1Present)
3515 {
3516 Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
3517 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
3518 }
3519
3520 return VINF_SUCCESS;
3521}
3522
3523
3524/**
3525 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
3526 * not.
3527 *
3528 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
3529 */
3530#ifdef VBOX_WITH_RAW_MODE_NOT_R0
3531# define IEMMISC_GET_EFL(a_pVCpu) ( CPUMRawGetEFlags(a_pVCpu) )
3532#else
3533# define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
3534#endif
3535
3536/**
3537 * Updates the EFLAGS in the correct manner wrt. PATM.
3538 *
3539 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
3540 * @param a_fEfl The new EFLAGS.
3541 */
3542#ifdef VBOX_WITH_RAW_MODE_NOT_R0
3543# define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) CPUMRawSetEFlags((a_pVCpu), a_fEfl)
3544#else
3545# define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
3546#endif
3547
3548
3549/** @} */
3550
3551/** @name Raising Exceptions.
3552 *
3553 * @{
3554 */
3555
3556
3557/**
3558 * Loads the specified stack far pointer from the TSS.
3559 *
3560 * @returns VBox strict status code.
3561 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3562 * @param uCpl The CPL to load the stack for.
3563 * @param pSelSS Where to return the new stack segment.
3564 * @param puEsp Where to return the new stack pointer.
3565 */
3566IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPU pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp)
3567{
3568 VBOXSTRICTRC rcStrict;
3569 Assert(uCpl < 4);
3570
3571 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
3572 switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
3573 {
3574 /*
3575 * 16-bit TSS (X86TSS16).
3576 */
3577 case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
3578 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
3579 {
3580 uint32_t off = uCpl * 4 + 2;
3581 if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
3582 {
3583 /** @todo check actual access pattern here. */
3584 uint32_t u32Tmp = 0; /* gcc maybe... */
3585 rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3586 if (rcStrict == VINF_SUCCESS)
3587 {
3588 *puEsp = RT_LOWORD(u32Tmp);
3589 *pSelSS = RT_HIWORD(u32Tmp);
3590 return VINF_SUCCESS;
3591 }
3592 }
3593 else
3594 {
3595 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
3596 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3597 }
3598 break;
3599 }
3600
3601 /*
3602 * 32-bit TSS (X86TSS32).
3603 */
3604 case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
3605 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
3606 {
3607 uint32_t off = uCpl * 8 + 4;
3608 if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
3609 {
3610/** @todo check actual access pattern here. */
3611 uint64_t u64Tmp;
3612 rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3613 if (rcStrict == VINF_SUCCESS)
3614 {
3615 *puEsp = u64Tmp & UINT32_MAX;
3616 *pSelSS = (RTSEL)(u64Tmp >> 32);
3617 return VINF_SUCCESS;
3618 }
3619 }
3620 else
3621 {
3622 Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
3623 rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3624 }
3625 break;
3626 }
3627
3628 default:
3629 AssertFailed();
3630 rcStrict = VERR_IEM_IPE_4;
3631 break;
3632 }
3633
3634 *puEsp = 0; /* make gcc happy */
3635 *pSelSS = 0; /* make gcc happy */
3636 return rcStrict;
3637}
3638
3639
3640/**
3641 * Loads the specified stack pointer from the 64-bit TSS.
3642 *
3643 * @returns VBox strict status code.
3644 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3645 * @param uCpl The CPL to load the stack for.
3646 * @param uIst The interrupt stack table index, 0 if to use uCpl.
3647 * @param puRsp Where to return the new stack pointer.
3648 */
3649IEM_STATIC VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPU pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp)
3650{
3651 Assert(uCpl < 4);
3652 Assert(uIst < 8);
3653 *puRsp = 0; /* make gcc happy */
3654
3655 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
3656 AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
3657
3658 uint32_t off;
3659 if (uIst)
3660 off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
3661 else
3662 off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
3663 if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
3664 {
3665 Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
3666 return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
3667 }
3668
3669 return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
3670}
3671
3672
3673/**
3674 * Adjust the CPU state according to the exception being raised.
3675 *
3676 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3677 * @param u8Vector The exception that has been raised.
3678 */
3679DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPU pVCpu, uint8_t u8Vector)
3680{
3681 switch (u8Vector)
3682 {
3683 case X86_XCPT_DB:
3684 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
3685 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
3686 break;
3687 /** @todo Read the AMD and Intel exception reference... */
3688 }
3689}
3690
3691
3692/**
3693 * Implements exceptions and interrupts for real mode.
3694 *
3695 * @returns VBox strict status code.
3696 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3697 * @param cbInstr The number of bytes to offset rIP by in the return
3698 * address.
3699 * @param u8Vector The interrupt / exception vector number.
3700 * @param fFlags The flags.
3701 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3702 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3703 */
3704IEM_STATIC VBOXSTRICTRC
3705iemRaiseXcptOrIntInRealMode(PVMCPU pVCpu,
3706 uint8_t cbInstr,
3707 uint8_t u8Vector,
3708 uint32_t fFlags,
3709 uint16_t uErr,
3710 uint64_t uCr2)
3711{
3712 NOREF(uErr); NOREF(uCr2);
3713 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3714
3715 /*
3716 * Read the IDT entry.
3717 */
3718 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
3719 {
3720 Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
3721 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
3722 }
3723 RTFAR16 Idte;
3724 VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector);
3725 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3726 {
3727 Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
3728 return rcStrict;
3729 }
3730
3731 /*
3732 * Push the stack frame.
3733 */
3734 uint16_t *pu16Frame;
3735 uint64_t uNewRsp;
3736 rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, (void **)&pu16Frame, &uNewRsp);
3737 if (rcStrict != VINF_SUCCESS)
3738 return rcStrict;
3739
3740 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
3741#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
3742 AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
3743 if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
3744 fEfl |= UINT16_C(0xf000);
3745#endif
3746 pu16Frame[2] = (uint16_t)fEfl;
3747 pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
3748 pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
3749 rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
3750 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
3751 return rcStrict;
3752
3753 /*
3754 * Load the vector address into cs:ip and make exception specific state
3755 * adjustments.
3756 */
3757 pVCpu->cpum.GstCtx.cs.Sel = Idte.sel;
3758 pVCpu->cpum.GstCtx.cs.ValidSel = Idte.sel;
3759 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
3760 pVCpu->cpum.GstCtx.cs.u64Base = (uint32_t)Idte.sel << 4;
3761 /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
3762 pVCpu->cpum.GstCtx.rip = Idte.off;
3763 fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC);
3764 IEMMISC_SET_EFL(pVCpu, fEfl);
3765
3766 /** @todo do we actually do this in real mode? */
3767 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
3768 iemRaiseXcptAdjustState(pVCpu, u8Vector);
3769
3770 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
3771}
3772
3773
3774/**
3775 * Loads a NULL data selector into when coming from V8086 mode.
3776 *
3777 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3778 * @param pSReg Pointer to the segment register.
3779 */
3780IEM_STATIC void iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPU pVCpu, PCPUMSELREG pSReg)
3781{
3782 pSReg->Sel = 0;
3783 pSReg->ValidSel = 0;
3784 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3785 {
3786 /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
3787 pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D;
3788 pSReg->Attr.u |= X86DESCATTR_UNUSABLE;
3789 }
3790 else
3791 {
3792 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3793 /** @todo check this on AMD-V */
3794 pSReg->u64Base = 0;
3795 pSReg->u32Limit = 0;
3796 }
3797}
3798
3799
3800/**
3801 * Loads a segment selector during a task switch in V8086 mode.
3802 *
3803 * @param pSReg Pointer to the segment register.
3804 * @param uSel The selector value to load.
3805 */
3806IEM_STATIC void iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
3807{
3808 /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3809 pSReg->Sel = uSel;
3810 pSReg->ValidSel = uSel;
3811 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3812 pSReg->u64Base = uSel << 4;
3813 pSReg->u32Limit = 0xffff;
3814 pSReg->Attr.u = 0xf3;
3815}
3816
3817
3818/**
3819 * Loads a NULL data selector into a selector register, both the hidden and
3820 * visible parts, in protected mode.
3821 *
3822 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3823 * @param pSReg Pointer to the segment register.
3824 * @param uRpl The RPL.
3825 */
3826IEM_STATIC void iemHlpLoadNullDataSelectorProt(PVMCPU pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
3827{
3828 /** @todo Testcase: write a testcase checking what happends when loading a NULL
3829 * data selector in protected mode. */
3830 pSReg->Sel = uRpl;
3831 pSReg->ValidSel = uRpl;
3832 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3833 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3834 {
3835 /* VT-x (Intel 3960x) observed doing something like this. */
3836 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
3837 pSReg->u32Limit = UINT32_MAX;
3838 pSReg->u64Base = 0;
3839 }
3840 else
3841 {
3842 pSReg->Attr.u = X86DESCATTR_UNUSABLE;
3843 pSReg->u32Limit = 0;
3844 pSReg->u64Base = 0;
3845 }
3846}
3847
3848
3849/**
3850 * Loads a segment selector during a task switch in protected mode.
3851 *
3852 * In this task switch scenario, we would throw \#TS exceptions rather than
3853 * \#GPs.
3854 *
3855 * @returns VBox strict status code.
3856 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3857 * @param pSReg Pointer to the segment register.
3858 * @param uSel The new selector value.
3859 *
3860 * @remarks This does _not_ handle CS or SS.
3861 * @remarks This expects pVCpu->iem.s.uCpl to be up to date.
3862 */
3863IEM_STATIC VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPU pVCpu, PCPUMSELREG pSReg, uint16_t uSel)
3864{
3865 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
3866
3867 /* Null data selector. */
3868 if (!(uSel & X86_SEL_MASK_OFF_RPL))
3869 {
3870 iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
3871 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
3872 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
3873 return VINF_SUCCESS;
3874 }
3875
3876 /* Fetch the descriptor. */
3877 IEMSELDESC Desc;
3878 VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
3879 if (rcStrict != VINF_SUCCESS)
3880 {
3881 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
3882 VBOXSTRICTRC_VAL(rcStrict)));
3883 return rcStrict;
3884 }
3885
3886 /* Must be a data segment or readable code segment. */
3887 if ( !Desc.Legacy.Gen.u1DescType
3888 || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3889 {
3890 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
3891 Desc.Legacy.Gen.u4Type));
3892 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3893 }
3894
3895 /* Check privileges for data segments and non-conforming code segments. */
3896 if ( (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
3897 != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
3898 {
3899 /* The RPL and the new CPL must be less than or equal to the DPL. */
3900 if ( (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
3901 || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
3902 {
3903 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
3904 uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
3905 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3906 }
3907 }
3908
3909 /* Is it there? */
3910 if (!Desc.Legacy.Gen.u1Present)
3911 {
3912 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
3913 return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
3914 }
3915
3916 /* The base and limit. */
3917 uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
3918 uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
3919
3920 /*
3921 * Ok, everything checked out fine. Now set the accessed bit before
3922 * committing the result into the registers.
3923 */
3924 if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
3925 {
3926 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
3927 if (rcStrict != VINF_SUCCESS)
3928 return rcStrict;
3929 Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
3930 }
3931
3932 /* Commit */
3933 pSReg->Sel = uSel;
3934 pSReg->Attr.u = X86DESC_GET_HID_ATTR(&Desc.Legacy);
3935 pSReg->u32Limit = cbLimit;
3936 pSReg->u64Base = u64Base; /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
3937 pSReg->ValidSel = uSel;
3938 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
3939 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
3940 pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
3941
3942 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
3943 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
3944 return VINF_SUCCESS;
3945}
3946
3947
3948/**
3949 * Performs a task switch.
3950 *
3951 * If the task switch is the result of a JMP, CALL or IRET instruction, the
3952 * caller is responsible for performing the necessary checks (like DPL, TSS
3953 * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
3954 * reference for JMP, CALL, IRET.
3955 *
3956 * If the task switch is the due to a software interrupt or hardware exception,
3957 * the caller is responsible for validating the TSS selector and descriptor. See
3958 * Intel Instruction reference for INT n.
3959 *
3960 * @returns VBox strict status code.
3961 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3962 * @param enmTaskSwitch What caused this task switch.
3963 * @param uNextEip The EIP effective after the task switch.
3964 * @param fFlags The flags, see IEM_XCPT_FLAGS_XXX.
3965 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
3966 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
3967 * @param SelTSS The TSS selector of the new task.
3968 * @param pNewDescTSS Pointer to the new TSS descriptor.
3969 */
3970IEM_STATIC VBOXSTRICTRC
3971iemTaskSwitch(PVMCPU pVCpu,
3972 IEMTASKSWITCH enmTaskSwitch,
3973 uint32_t uNextEip,
3974 uint32_t fFlags,
3975 uint16_t uErr,
3976 uint64_t uCr2,
3977 RTSEL SelTSS,
3978 PIEMSELDESC pNewDescTSS)
3979{
3980 Assert(!IEM_IS_REAL_MODE(pVCpu));
3981 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
3982 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
3983
3984 uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
3985 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
3986 || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
3987 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
3988 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
3989
3990 bool const fIsNewTSS386 = ( uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
3991 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
3992
3993 Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
3994 fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
3995
3996 /* Update CR2 in case it's a page-fault. */
3997 /** @todo This should probably be done much earlier in IEM/PGM. See
3998 * @bugref{5653#c49}. */
3999 if (fFlags & IEM_XCPT_FLAGS_CR2)
4000 pVCpu->cpum.GstCtx.cr2 = uCr2;
4001
4002 /*
4003 * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
4004 * subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
4005 */
4006 uint32_t const uNewTSSLimit = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
4007 uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
4008 if (uNewTSSLimit < uNewTSSLimitMin)
4009 {
4010 Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
4011 enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
4012 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
4013 }
4014
4015 /*
4016 * The SVM nested-guest intercept for task-switch takes priority over all exceptions
4017 * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
4018 */
4019 if (IEM_IS_SVM_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
4020 {
4021 uint32_t const uExitInfo1 = SelTSS;
4022 uint32_t uExitInfo2 = uErr;
4023 switch (enmTaskSwitch)
4024 {
4025 case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break;
4026 case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break;
4027 default: break;
4028 }
4029 if (fFlags & IEM_XCPT_FLAGS_ERR)
4030 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
4031 if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
4032 uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
4033
4034 Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
4035 IEM_RETURN_SVM_VMEXIT(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
4036 RT_NOREF2(uExitInfo1, uExitInfo2);
4037 }
4038 /** @todo Nested-VMX task-switch intercept. */
4039
4040 /*
4041 * Check the current TSS limit. The last written byte to the current TSS during the
4042 * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
4043 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
4044 *
4045 * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
4046 * end up with smaller than "legal" TSS limits.
4047 */
4048 uint32_t const uCurTSSLimit = pVCpu->cpum.GstCtx.tr.u32Limit;
4049 uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
4050 if (uCurTSSLimit < uCurTSSLimitMin)
4051 {
4052 Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
4053 enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
4054 return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
4055 }
4056
4057 /*
4058 * Verify that the new TSS can be accessed and map it. Map only the required contents
4059 * and not the entire TSS.
4060 */
4061 void *pvNewTSS;
4062 uint32_t cbNewTSS = uNewTSSLimitMin + 1;
4063 RTGCPTR GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
4064 AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
4065 /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
4066 * not perform correct translation if this happens. See Intel spec. 7.2.1
4067 * "Task-State Segment" */
4068 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW);
4069 if (rcStrict != VINF_SUCCESS)
4070 {
4071 Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
4072 cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
4073 return rcStrict;
4074 }
4075
4076 /*
4077 * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
4078 */
4079 uint32_t u32EFlags = pVCpu->cpum.GstCtx.eflags.u32;
4080 if ( enmTaskSwitch == IEMTASKSWITCH_JUMP
4081 || enmTaskSwitch == IEMTASKSWITCH_IRET)
4082 {
4083 PX86DESC pDescCurTSS;
4084 rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX,
4085 pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
4086 if (rcStrict != VINF_SUCCESS)
4087 {
4088 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4089 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4090 return rcStrict;
4091 }
4092
4093 pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4094 rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
4095 if (rcStrict != VINF_SUCCESS)
4096 {
4097 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4098 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4099 return rcStrict;
4100 }
4101
4102 /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
4103 if (enmTaskSwitch == IEMTASKSWITCH_IRET)
4104 {
4105 Assert( uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
4106 || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
4107 u32EFlags &= ~X86_EFL_NT;
4108 }
4109 }
4110
4111 /*
4112 * Save the CPU state into the current TSS.
4113 */
4114 RTGCPTR GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
4115 if (GCPtrNewTSS == GCPtrCurTSS)
4116 {
4117 Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
4118 Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
4119 pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u32, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ldtr.Sel));
4120 }
4121 if (fIsNewTSS386)
4122 {
4123 /*
4124 * Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
4125 * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
4126 */
4127 void *pvCurTSS32;
4128 uint32_t offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
4129 uint32_t cbCurTSS = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
4130 AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
4131 rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
4132 if (rcStrict != VINF_SUCCESS)
4133 {
4134 Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
4135 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
4136 return rcStrict;
4137 }
4138
4139 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
4140 PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
4141 pCurTSS32->eip = uNextEip;
4142 pCurTSS32->eflags = u32EFlags;
4143 pCurTSS32->eax = pVCpu->cpum.GstCtx.eax;
4144 pCurTSS32->ecx = pVCpu->cpum.GstCtx.ecx;
4145 pCurTSS32->edx = pVCpu->cpum.GstCtx.edx;
4146 pCurTSS32->ebx = pVCpu->cpum.GstCtx.ebx;
4147 pCurTSS32->esp = pVCpu->cpum.GstCtx.esp;
4148 pCurTSS32->ebp = pVCpu->cpum.GstCtx.ebp;
4149 pCurTSS32->esi = pVCpu->cpum.GstCtx.esi;
4150 pCurTSS32->edi = pVCpu->cpum.GstCtx.edi;
4151 pCurTSS32->es = pVCpu->cpum.GstCtx.es.Sel;
4152 pCurTSS32->cs = pVCpu->cpum.GstCtx.cs.Sel;
4153 pCurTSS32->ss = pVCpu->cpum.GstCtx.ss.Sel;
4154 pCurTSS32->ds = pVCpu->cpum.GstCtx.ds.Sel;
4155 pCurTSS32->fs = pVCpu->cpum.GstCtx.fs.Sel;
4156 pCurTSS32->gs = pVCpu->cpum.GstCtx.gs.Sel;
4157
4158 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
4159 if (rcStrict != VINF_SUCCESS)
4160 {
4161 Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
4162 VBOXSTRICTRC_VAL(rcStrict)));
4163 return rcStrict;
4164 }
4165 }
4166 else
4167 {
4168 /*
4169 * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
4170 */
4171 void *pvCurTSS16;
4172 uint32_t offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
4173 uint32_t cbCurTSS = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
4174 AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
4175 rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW);
4176 if (rcStrict != VINF_SUCCESS)
4177 {
4178 Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
4179 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
4180 return rcStrict;
4181 }
4182
4183 /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
4184 PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
4185 pCurTSS16->ip = uNextEip;
4186 pCurTSS16->flags = u32EFlags;
4187 pCurTSS16->ax = pVCpu->cpum.GstCtx.ax;
4188 pCurTSS16->cx = pVCpu->cpum.GstCtx.cx;
4189 pCurTSS16->dx = pVCpu->cpum.GstCtx.dx;
4190 pCurTSS16->bx = pVCpu->cpum.GstCtx.bx;
4191 pCurTSS16->sp = pVCpu->cpum.GstCtx.sp;
4192 pCurTSS16->bp = pVCpu->cpum.GstCtx.bp;
4193 pCurTSS16->si = pVCpu->cpum.GstCtx.si;
4194 pCurTSS16->di = pVCpu->cpum.GstCtx.di;
4195 pCurTSS16->es = pVCpu->cpum.GstCtx.es.Sel;
4196 pCurTSS16->cs = pVCpu->cpum.GstCtx.cs.Sel;
4197 pCurTSS16->ss = pVCpu->cpum.GstCtx.ss.Sel;
4198 pCurTSS16->ds = pVCpu->cpum.GstCtx.ds.Sel;
4199
4200 rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
4201 if (rcStrict != VINF_SUCCESS)
4202 {
4203 Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
4204 VBOXSTRICTRC_VAL(rcStrict)));
4205 return rcStrict;
4206 }
4207 }
4208
4209 /*
4210 * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
4211 */
4212 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
4213 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
4214 {
4215 /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
4216 PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
4217 pNewTSS->selPrev = pVCpu->cpum.GstCtx.tr.Sel;
4218 }
4219
4220 /*
4221 * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
4222 * it's done further below with error handling (e.g. CR3 changes will go through PGM).
4223 */
4224 uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
4225 uint16_t uNewES, uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
4226 bool fNewDebugTrap;
4227 if (fIsNewTSS386)
4228 {
4229 PX86TSS32 pNewTSS32 = (PX86TSS32)pvNewTSS;
4230 uNewCr3 = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
4231 uNewEip = pNewTSS32->eip;
4232 uNewEflags = pNewTSS32->eflags;
4233 uNewEax = pNewTSS32->eax;
4234 uNewEcx = pNewTSS32->ecx;
4235 uNewEdx = pNewTSS32->edx;
4236 uNewEbx = pNewTSS32->ebx;
4237 uNewEsp = pNewTSS32->esp;
4238 uNewEbp = pNewTSS32->ebp;
4239 uNewEsi = pNewTSS32->esi;
4240 uNewEdi = pNewTSS32->edi;
4241 uNewES = pNewTSS32->es;
4242 uNewCS = pNewTSS32->cs;
4243 uNewSS = pNewTSS32->ss;
4244 uNewDS = pNewTSS32->ds;
4245 uNewFS = pNewTSS32->fs;
4246 uNewGS = pNewTSS32->gs;
4247 uNewLdt = pNewTSS32->selLdt;
4248 fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
4249 }
4250 else
4251 {
4252 PX86TSS16 pNewTSS16 = (PX86TSS16)pvNewTSS;
4253 uNewCr3 = 0;
4254 uNewEip = pNewTSS16->ip;
4255 uNewEflags = pNewTSS16->flags;
4256 uNewEax = UINT32_C(0xffff0000) | pNewTSS16->ax;
4257 uNewEcx = UINT32_C(0xffff0000) | pNewTSS16->cx;
4258 uNewEdx = UINT32_C(0xffff0000) | pNewTSS16->dx;
4259 uNewEbx = UINT32_C(0xffff0000) | pNewTSS16->bx;
4260 uNewEsp = UINT32_C(0xffff0000) | pNewTSS16->sp;
4261 uNewEbp = UINT32_C(0xffff0000) | pNewTSS16->bp;
4262 uNewEsi = UINT32_C(0xffff0000) | pNewTSS16->si;
4263 uNewEdi = UINT32_C(0xffff0000) | pNewTSS16->di;
4264 uNewES = pNewTSS16->es;
4265 uNewCS = pNewTSS16->cs;
4266 uNewSS = pNewTSS16->ss;
4267 uNewDS = pNewTSS16->ds;
4268 uNewFS = 0;
4269 uNewGS = 0;
4270 uNewLdt = pNewTSS16->selLdt;
4271 fNewDebugTrap = false;
4272 }
4273
4274 if (GCPtrNewTSS == GCPtrCurTSS)
4275 Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
4276 uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
4277
4278 /*
4279 * We're done accessing the new TSS.
4280 */
4281 rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
4282 if (rcStrict != VINF_SUCCESS)
4283 {
4284 Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
4285 return rcStrict;
4286 }
4287
4288 /*
4289 * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
4290 */
4291 if (enmTaskSwitch != IEMTASKSWITCH_IRET)
4292 {
4293 rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX,
4294 pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW);
4295 if (rcStrict != VINF_SUCCESS)
4296 {
4297 Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4298 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4299 return rcStrict;
4300 }
4301
4302 /* Check that the descriptor indicates the new TSS is available (not busy). */
4303 AssertMsg( pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
4304 || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
4305 ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
4306
4307 pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4308 rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
4309 if (rcStrict != VINF_SUCCESS)
4310 {
4311 Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
4312 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
4313 return rcStrict;
4314 }
4315 }
4316
4317 /*
4318 * From this point on, we're technically in the new task. We will defer exceptions
4319 * until the completion of the task switch but before executing any instructions in the new task.
4320 */
4321 pVCpu->cpum.GstCtx.tr.Sel = SelTSS;
4322 pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
4323 pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
4324 pVCpu->cpum.GstCtx.tr.Attr.u = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
4325 pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
4326 pVCpu->cpum.GstCtx.tr.u64Base = X86DESC_BASE(&pNewDescTSS->Legacy);
4327 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
4328
4329 /* Set the busy bit in TR. */
4330 pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
4331 /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
4332 if ( enmTaskSwitch == IEMTASKSWITCH_CALL
4333 || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
4334 {
4335 uNewEflags |= X86_EFL_NT;
4336 }
4337
4338 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL; /** @todo Should we clear DR7.LE bit too? */
4339 pVCpu->cpum.GstCtx.cr0 |= X86_CR0_TS;
4340 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
4341
4342 pVCpu->cpum.GstCtx.eip = uNewEip;
4343 pVCpu->cpum.GstCtx.eax = uNewEax;
4344 pVCpu->cpum.GstCtx.ecx = uNewEcx;
4345 pVCpu->cpum.GstCtx.edx = uNewEdx;
4346 pVCpu->cpum.GstCtx.ebx = uNewEbx;
4347 pVCpu->cpum.GstCtx.esp = uNewEsp;
4348 pVCpu->cpum.GstCtx.ebp = uNewEbp;
4349 pVCpu->cpum.GstCtx.esi = uNewEsi;
4350 pVCpu->cpum.GstCtx.edi = uNewEdi;
4351
4352 uNewEflags &= X86_EFL_LIVE_MASK;
4353 uNewEflags |= X86_EFL_RA1_MASK;
4354 IEMMISC_SET_EFL(pVCpu, uNewEflags);
4355
4356 /*
4357 * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
4358 * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
4359 * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
4360 */
4361 pVCpu->cpum.GstCtx.es.Sel = uNewES;
4362 pVCpu->cpum.GstCtx.es.Attr.u &= ~X86DESCATTR_P;
4363
4364 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
4365 pVCpu->cpum.GstCtx.cs.Attr.u &= ~X86DESCATTR_P;
4366
4367 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
4368 pVCpu->cpum.GstCtx.ss.Attr.u &= ~X86DESCATTR_P;
4369
4370 pVCpu->cpum.GstCtx.ds.Sel = uNewDS;
4371 pVCpu->cpum.GstCtx.ds.Attr.u &= ~X86DESCATTR_P;
4372
4373 pVCpu->cpum.GstCtx.fs.Sel = uNewFS;
4374 pVCpu->cpum.GstCtx.fs.Attr.u &= ~X86DESCATTR_P;
4375
4376 pVCpu->cpum.GstCtx.gs.Sel = uNewGS;
4377 pVCpu->cpum.GstCtx.gs.Attr.u &= ~X86DESCATTR_P;
4378 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
4379
4380 pVCpu->cpum.GstCtx.ldtr.Sel = uNewLdt;
4381 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_STALE;
4382 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
4383 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
4384
4385 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4386 {
4387 pVCpu->cpum.GstCtx.es.Attr.u |= X86DESCATTR_UNUSABLE;
4388 pVCpu->cpum.GstCtx.cs.Attr.u |= X86DESCATTR_UNUSABLE;
4389 pVCpu->cpum.GstCtx.ss.Attr.u |= X86DESCATTR_UNUSABLE;
4390 pVCpu->cpum.GstCtx.ds.Attr.u |= X86DESCATTR_UNUSABLE;
4391 pVCpu->cpum.GstCtx.fs.Attr.u |= X86DESCATTR_UNUSABLE;
4392 pVCpu->cpum.GstCtx.gs.Attr.u |= X86DESCATTR_UNUSABLE;
4393 pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE;
4394 }
4395
4396 /*
4397 * Switch CR3 for the new task.
4398 */
4399 if ( fIsNewTSS386
4400 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
4401 {
4402 /** @todo Should we update and flush TLBs only if CR3 value actually changes? */
4403 int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
4404 AssertRCSuccessReturn(rc, rc);
4405
4406 /* Inform PGM. */
4407 rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
4408 AssertRCReturn(rc, rc);
4409 /* ignore informational status codes */
4410
4411 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
4412 }
4413
4414 /*
4415 * Switch LDTR for the new task.
4416 */
4417 if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
4418 iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
4419 else
4420 {
4421 Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present); /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
4422
4423 IEMSELDESC DescNewLdt;
4424 rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
4425 if (rcStrict != VINF_SUCCESS)
4426 {
4427 Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
4428 uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
4429 return rcStrict;
4430 }
4431 if ( !DescNewLdt.Legacy.Gen.u1Present
4432 || DescNewLdt.Legacy.Gen.u1DescType
4433 || DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
4434 {
4435 Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
4436 uNewLdt, DescNewLdt.Legacy.u));
4437 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
4438 }
4439
4440 pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
4441 pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
4442 pVCpu->cpum.GstCtx.ldtr.u64Base = X86DESC_BASE(&DescNewLdt.Legacy);
4443 pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
4444 pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
4445 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
4446 pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
4447 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
4448 }
4449
4450 IEMSELDESC DescSS;
4451 if (IEM_IS_V86_MODE(pVCpu))
4452 {
4453 pVCpu->iem.s.uCpl = 3;
4454 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
4455 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
4456 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
4457 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
4458 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
4459 iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
4460
4461 /* quick fix: fake DescSS. */ /** @todo fix the code further down? */
4462 DescSS.Legacy.u = 0;
4463 DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
4464 DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
4465 DescSS.Legacy.Gen.u16BaseLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
4466 DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
4467 DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
4468 DescSS.Legacy.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
4469 DescSS.Legacy.Gen.u2Dpl = 3;
4470 }
4471 else
4472 {
4473 uint8_t uNewCpl = (uNewCS & X86_SEL_RPL);
4474
4475 /*
4476 * Load the stack segment for the new task.
4477 */
4478 if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
4479 {
4480 Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
4481 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4482 }
4483
4484 /* Fetch the descriptor. */
4485 rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
4486 if (rcStrict != VINF_SUCCESS)
4487 {
4488 Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
4489 VBOXSTRICTRC_VAL(rcStrict)));
4490 return rcStrict;
4491 }
4492
4493 /* SS must be a data segment and writable. */
4494 if ( !DescSS.Legacy.Gen.u1DescType
4495 || (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
4496 || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
4497 {
4498 Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
4499 uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
4500 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4501 }
4502
4503 /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
4504 if ( (uNewSS & X86_SEL_RPL) != uNewCpl
4505 || DescSS.Legacy.Gen.u2Dpl != uNewCpl)
4506 {
4507 Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
4508 uNewCpl));
4509 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4510 }
4511
4512 /* Is it there? */
4513 if (!DescSS.Legacy.Gen.u1Present)
4514 {
4515 Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
4516 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
4517 }
4518
4519 uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
4520 uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
4521
4522 /* Set the accessed bit before committing the result into SS. */
4523 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4524 {
4525 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
4526 if (rcStrict != VINF_SUCCESS)
4527 return rcStrict;
4528 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
4529 }
4530
4531 /* Commit SS. */
4532 pVCpu->cpum.GstCtx.ss.Sel = uNewSS;
4533 pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
4534 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
4535 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
4536 pVCpu->cpum.GstCtx.ss.u64Base = u64Base;
4537 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
4538 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
4539
4540 /* CPL has changed, update IEM before loading rest of segments. */
4541 pVCpu->iem.s.uCpl = uNewCpl;
4542
4543 /*
4544 * Load the data segments for the new task.
4545 */
4546 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
4547 if (rcStrict != VINF_SUCCESS)
4548 return rcStrict;
4549 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
4550 if (rcStrict != VINF_SUCCESS)
4551 return rcStrict;
4552 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
4553 if (rcStrict != VINF_SUCCESS)
4554 return rcStrict;
4555 rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
4556 if (rcStrict != VINF_SUCCESS)
4557 return rcStrict;
4558
4559 /*
4560 * Load the code segment for the new task.
4561 */
4562 if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
4563 {
4564 Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
4565 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4566 }
4567
4568 /* Fetch the descriptor. */
4569 IEMSELDESC DescCS;
4570 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
4571 if (rcStrict != VINF_SUCCESS)
4572 {
4573 Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
4574 return rcStrict;
4575 }
4576
4577 /* CS must be a code segment. */
4578 if ( !DescCS.Legacy.Gen.u1DescType
4579 || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
4580 {
4581 Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
4582 DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
4583 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4584 }
4585
4586 /* For conforming CS, DPL must be less than or equal to the RPL. */
4587 if ( (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
4588 && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
4589 {
4590 Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
4591 DescCS.Legacy.Gen.u2Dpl));
4592 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4593 }
4594
4595 /* For non-conforming CS, DPL must match RPL. */
4596 if ( !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
4597 && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
4598 {
4599 Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
4600 DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
4601 return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4602 }
4603
4604 /* Is it there? */
4605 if (!DescCS.Legacy.Gen.u1Present)
4606 {
4607 Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
4608 return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
4609 }
4610
4611 cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
4612 u64Base = X86DESC_BASE(&DescCS.Legacy);
4613
4614 /* Set the accessed bit before committing the result into CS. */
4615 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
4616 {
4617 rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
4618 if (rcStrict != VINF_SUCCESS)
4619 return rcStrict;
4620 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
4621 }
4622
4623 /* Commit CS. */
4624 pVCpu->cpum.GstCtx.cs.Sel = uNewCS;
4625 pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
4626 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
4627 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
4628 pVCpu->cpum.GstCtx.cs.u64Base = u64Base;
4629 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
4630 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
4631 }
4632
4633 /** @todo Debug trap. */
4634 if (fIsNewTSS386 && fNewDebugTrap)
4635 Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
4636
4637 /*
4638 * Construct the error code masks based on what caused this task switch.
4639 * See Intel Instruction reference for INT.
4640 */
4641 uint16_t uExt;
4642 if ( enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
4643 && !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT))
4644 {
4645 uExt = 1;
4646 }
4647 else
4648 uExt = 0;
4649
4650 /*
4651 * Push any error code on to the new stack.
4652 */
4653 if (fFlags & IEM_XCPT_FLAGS_ERR)
4654 {
4655 Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
4656 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
4657 uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
4658
4659 /* Check that there is sufficient space on the stack. */
4660 /** @todo Factor out segment limit checking for normal/expand down segments
4661 * into a separate function. */
4662 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
4663 {
4664 if ( pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
4665 || pVCpu->cpum.GstCtx.esp < cbStackFrame)
4666 {
4667 /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
4668 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
4669 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
4670 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
4671 }
4672 }
4673 else
4674 {
4675 if ( pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
4676 || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
4677 {
4678 Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
4679 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
4680 return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
4681 }
4682 }
4683
4684
4685 if (fIsNewTSS386)
4686 rcStrict = iemMemStackPushU32(pVCpu, uErr);
4687 else
4688 rcStrict = iemMemStackPushU16(pVCpu, uErr);
4689 if (rcStrict != VINF_SUCCESS)
4690 {
4691 Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
4692 fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
4693 return rcStrict;
4694 }
4695 }
4696
4697 /* Check the new EIP against the new CS limit. */
4698 if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
4699 {
4700 Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
4701 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
4702 /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
4703 return iemRaiseGeneralProtectionFault(pVCpu, uExt);
4704 }
4705
4706 Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.ss.Sel));
4707 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
4708}
4709
4710
4711/**
4712 * Implements exceptions and interrupts for protected mode.
4713 *
4714 * @returns VBox strict status code.
4715 * @param pVCpu The cross context virtual CPU structure of the calling thread.
4716 * @param cbInstr The number of bytes to offset rIP by in the return
4717 * address.
4718 * @param u8Vector The interrupt / exception vector number.
4719 * @param fFlags The flags.
4720 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
4721 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
4722 */
4723IEM_STATIC VBOXSTRICTRC
4724iemRaiseXcptOrIntInProtMode(PVMCPU pVCpu,
4725 uint8_t cbInstr,
4726 uint8_t u8Vector,
4727 uint32_t fFlags,
4728 uint16_t uErr,
4729 uint64_t uCr2)
4730{
4731 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
4732
4733 /*
4734 * Read the IDT entry.
4735 */
4736 if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
4737 {
4738 Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
4739 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4740 }
4741 X86DESC Idte;
4742 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
4743 pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
4744 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
4745 {
4746 Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
4747 return rcStrict;
4748 }
4749 Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
4750 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
4751 Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
4752
4753 /*
4754 * Check the descriptor type, DPL and such.
4755 * ASSUMES this is done in the same order as described for call-gate calls.
4756 */
4757 if (Idte.Gate.u1DescType)
4758 {
4759 Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
4760 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4761 }
4762 bool fTaskGate = false;
4763 uint8_t f32BitGate = true;
4764 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
4765 switch (Idte.Gate.u4Type)
4766 {
4767 case X86_SEL_TYPE_SYS_UNDEFINED:
4768 case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
4769 case X86_SEL_TYPE_SYS_LDT:
4770 case X86_SEL_TYPE_SYS_286_TSS_BUSY:
4771 case X86_SEL_TYPE_SYS_286_CALL_GATE:
4772 case X86_SEL_TYPE_SYS_UNDEFINED2:
4773 case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
4774 case X86_SEL_TYPE_SYS_UNDEFINED3:
4775 case X86_SEL_TYPE_SYS_386_TSS_BUSY:
4776 case X86_SEL_TYPE_SYS_386_CALL_GATE:
4777 case X86_SEL_TYPE_SYS_UNDEFINED4:
4778 {
4779 /** @todo check what actually happens when the type is wrong...
4780 * esp. call gates. */
4781 Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
4782 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4783 }
4784
4785 case X86_SEL_TYPE_SYS_286_INT_GATE:
4786 f32BitGate = false;
4787 RT_FALL_THRU();
4788 case X86_SEL_TYPE_SYS_386_INT_GATE:
4789 fEflToClear |= X86_EFL_IF;
4790 break;
4791
4792 case X86_SEL_TYPE_SYS_TASK_GATE:
4793 fTaskGate = true;
4794#ifndef IEM_IMPLEMENTS_TASKSWITCH
4795 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
4796#endif
4797 break;
4798
4799 case X86_SEL_TYPE_SYS_286_TRAP_GATE:
4800 f32BitGate = false;
4801 case X86_SEL_TYPE_SYS_386_TRAP_GATE:
4802 break;
4803
4804 IEM_NOT_REACHED_DEFAULT_CASE_RET();
4805 }
4806
4807 /* Check DPL against CPL if applicable. */
4808 if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
4809 {
4810 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
4811 {
4812 Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
4813 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4814 }
4815 }
4816
4817 /* Is it there? */
4818 if (!Idte.Gate.u1Present)
4819 {
4820 Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
4821 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
4822 }
4823
4824 /* Is it a task-gate? */
4825 if (fTaskGate)
4826 {
4827 /*
4828 * Construct the error code masks based on what caused this task switch.
4829 * See Intel Instruction reference for INT.
4830 */
4831 uint16_t const uExt = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? 0 : 1;
4832 uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
4833 RTSEL SelTSS = Idte.Gate.u16Sel;
4834
4835 /*
4836 * Fetch the TSS descriptor in the GDT.
4837 */
4838 IEMSELDESC DescTSS;
4839 rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt);
4840 if (rcStrict != VINF_SUCCESS)
4841 {
4842 Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
4843 VBOXSTRICTRC_VAL(rcStrict)));
4844 return rcStrict;
4845 }
4846
4847 /* The TSS descriptor must be a system segment and be available (not busy). */
4848 if ( DescTSS.Legacy.Gen.u1DescType
4849 || ( DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
4850 && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
4851 {
4852 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
4853 u8Vector, SelTSS, DescTSS.Legacy.au64));
4854 return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt);
4855 }
4856
4857 /* The TSS must be present. */
4858 if (!DescTSS.Legacy.Gen.u1Present)
4859 {
4860 Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
4861 return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt);
4862 }
4863
4864 /* Do the actual task switch. */
4865 return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT, pVCpu->cpum.GstCtx.eip, fFlags, uErr, uCr2, SelTSS, &DescTSS);
4866 }
4867
4868 /* A null CS is bad. */
4869 RTSEL NewCS = Idte.Gate.u16Sel;
4870 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
4871 {
4872 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
4873 return iemRaiseGeneralProtectionFault0(pVCpu);
4874 }
4875
4876 /* Fetch the descriptor for the new CS. */
4877 IEMSELDESC DescCS;
4878 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
4879 if (rcStrict != VINF_SUCCESS)
4880 {
4881 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
4882 return rcStrict;
4883 }
4884
4885 /* Must be a code segment. */
4886 if (!DescCS.Legacy.Gen.u1DescType)
4887 {
4888 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
4889 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4890 }
4891 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
4892 {
4893 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
4894 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4895 }
4896
4897 /* Don't allow lowering the privilege level. */
4898 /** @todo Does the lowering of privileges apply to software interrupts
4899 * only? This has bearings on the more-privileged or
4900 * same-privilege stack behavior further down. A testcase would
4901 * be nice. */
4902 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
4903 {
4904 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
4905 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
4906 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
4907 }
4908
4909 /* Make sure the selector is present. */
4910 if (!DescCS.Legacy.Gen.u1Present)
4911 {
4912 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
4913 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
4914 }
4915
4916 /* Check the new EIP against the new CS limit. */
4917 uint32_t const uNewEip = Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
4918 || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
4919 ? Idte.Gate.u16OffsetLow
4920 : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
4921 uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
4922 if (uNewEip > cbLimitCS)
4923 {
4924 Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
4925 u8Vector, uNewEip, cbLimitCS, NewCS));
4926 return iemRaiseGeneralProtectionFault(pVCpu, 0);
4927 }
4928 Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
4929
4930 /* Calc the flag image to push. */
4931 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
4932 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
4933 fEfl &= ~X86_EFL_RF;
4934 else
4935 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
4936
4937 /* From V8086 mode only go to CPL 0. */
4938 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
4939 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
4940 if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
4941 {
4942 Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
4943 return iemRaiseGeneralProtectionFault(pVCpu, 0);
4944 }
4945
4946 /*
4947 * If the privilege level changes, we need to get a new stack from the TSS.
4948 * This in turns means validating the new SS and ESP...
4949 */
4950 if (uNewCpl != pVCpu->iem.s.uCpl)
4951 {
4952 RTSEL NewSS;
4953 uint32_t uNewEsp;
4954 rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
4955 if (rcStrict != VINF_SUCCESS)
4956 return rcStrict;
4957
4958 IEMSELDESC DescSS;
4959 rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
4960 if (rcStrict != VINF_SUCCESS)
4961 return rcStrict;
4962 /* If the new SS is 16-bit, we are only going to use SP, not ESP. */
4963 if (!DescSS.Legacy.Gen.u1DefBig)
4964 {
4965 Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
4966 uNewEsp = (uint16_t)uNewEsp;
4967 }
4968
4969 Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
4970
4971 /* Check that there is sufficient space for the stack frame. */
4972 uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
4973 uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
4974 ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
4975 : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
4976
4977 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
4978 {
4979 if ( uNewEsp - 1 > cbLimitSS
4980 || uNewEsp < cbStackFrame)
4981 {
4982 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
4983 u8Vector, NewSS, uNewEsp, cbStackFrame));
4984 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
4985 }
4986 }
4987 else
4988 {
4989 if ( uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
4990 || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
4991 {
4992 Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
4993 u8Vector, NewSS, uNewEsp, cbStackFrame));
4994 return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
4995 }
4996 }
4997
4998 /*
4999 * Start making changes.
5000 */
5001
5002 /* Set the new CPL so that stack accesses use it. */
5003 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
5004 pVCpu->iem.s.uCpl = uNewCpl;
5005
5006 /* Create the stack frame. */
5007 RTPTRUNION uStackFrame;
5008 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
5009 uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy), IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
5010 if (rcStrict != VINF_SUCCESS)
5011 return rcStrict;
5012 void * const pvStackFrame = uStackFrame.pv;
5013 if (f32BitGate)
5014 {
5015 if (fFlags & IEM_XCPT_FLAGS_ERR)
5016 *uStackFrame.pu32++ = uErr;
5017 uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5018 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
5019 uStackFrame.pu32[2] = fEfl;
5020 uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
5021 uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
5022 Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
5023 if (fEfl & X86_EFL_VM)
5024 {
5025 uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
5026 uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
5027 uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
5028 uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
5029 uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
5030 }
5031 }
5032 else
5033 {
5034 if (fFlags & IEM_XCPT_FLAGS_ERR)
5035 *uStackFrame.pu16++ = uErr;
5036 uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
5037 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
5038 uStackFrame.pu16[2] = fEfl;
5039 uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
5040 uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
5041 Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
5042 if (fEfl & X86_EFL_VM)
5043 {
5044 uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
5045 uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
5046 uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
5047 uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
5048 uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
5049 }
5050 }
5051 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
5052 if (rcStrict != VINF_SUCCESS)
5053 return rcStrict;
5054
5055 /* Mark the selectors 'accessed' (hope this is the correct time). */
5056 /** @todo testcase: excatly _when_ are the accessed bits set - before or
5057 * after pushing the stack frame? (Write protect the gdt + stack to
5058 * find out.) */
5059 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5060 {
5061 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5062 if (rcStrict != VINF_SUCCESS)
5063 return rcStrict;
5064 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5065 }
5066
5067 if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5068 {
5069 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
5070 if (rcStrict != VINF_SUCCESS)
5071 return rcStrict;
5072 DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5073 }
5074
5075 /*
5076 * Start comitting the register changes (joins with the DPL=CPL branch).
5077 */
5078 pVCpu->cpum.GstCtx.ss.Sel = NewSS;
5079 pVCpu->cpum.GstCtx.ss.ValidSel = NewSS;
5080 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
5081 pVCpu->cpum.GstCtx.ss.u32Limit = cbLimitSS;
5082 pVCpu->cpum.GstCtx.ss.u64Base = X86DESC_BASE(&DescSS.Legacy);
5083 pVCpu->cpum.GstCtx.ss.Attr.u = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
5084 /** @todo When coming from 32-bit code and operating with a 16-bit TSS and
5085 * 16-bit handler, the high word of ESP remains unchanged (i.e. only
5086 * SP is loaded).
5087 * Need to check the other combinations too:
5088 * - 16-bit TSS, 32-bit handler
5089 * - 32-bit TSS, 16-bit handler */
5090 if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
5091 pVCpu->cpum.GstCtx.sp = (uint16_t)(uNewEsp - cbStackFrame);
5092 else
5093 pVCpu->cpum.GstCtx.rsp = uNewEsp - cbStackFrame;
5094
5095 if (fEfl & X86_EFL_VM)
5096 {
5097 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
5098 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
5099 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
5100 iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
5101 }
5102 }
5103 /*
5104 * Same privilege, no stack change and smaller stack frame.
5105 */
5106 else
5107 {
5108 uint64_t uNewRsp;
5109 RTPTRUNION uStackFrame;
5110 uint8_t const cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
5111 rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, &uStackFrame.pv, &uNewRsp);
5112 if (rcStrict != VINF_SUCCESS)
5113 return rcStrict;
5114 void * const pvStackFrame = uStackFrame.pv;
5115
5116 if (f32BitGate)
5117 {
5118 if (fFlags & IEM_XCPT_FLAGS_ERR)
5119 *uStackFrame.pu32++ = uErr;
5120 uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5121 uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
5122 uStackFrame.pu32[2] = fEfl;
5123 }
5124 else
5125 {
5126 if (fFlags & IEM_XCPT_FLAGS_ERR)
5127 *uStackFrame.pu16++ = uErr;
5128 uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
5129 uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
5130 uStackFrame.pu16[2] = fEfl;
5131 }
5132 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
5133 if (rcStrict != VINF_SUCCESS)
5134 return rcStrict;
5135
5136 /* Mark the CS selector as 'accessed'. */
5137 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5138 {
5139 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5140 if (rcStrict != VINF_SUCCESS)
5141 return rcStrict;
5142 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5143 }
5144
5145 /*
5146 * Start committing the register changes (joins with the other branch).
5147 */
5148 pVCpu->cpum.GstCtx.rsp = uNewRsp;
5149 }
5150
5151 /* ... register committing continues. */
5152 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5153 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5154 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
5155 pVCpu->cpum.GstCtx.cs.u32Limit = cbLimitCS;
5156 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
5157 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
5158
5159 pVCpu->cpum.GstCtx.rip = uNewEip; /* (The entire register is modified, see pe16_32 bs3kit tests.) */
5160 fEfl &= ~fEflToClear;
5161 IEMMISC_SET_EFL(pVCpu, fEfl);
5162
5163 if (fFlags & IEM_XCPT_FLAGS_CR2)
5164 pVCpu->cpum.GstCtx.cr2 = uCr2;
5165
5166 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
5167 iemRaiseXcptAdjustState(pVCpu, u8Vector);
5168
5169 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
5170}
5171
5172
5173/**
5174 * Implements exceptions and interrupts for long mode.
5175 *
5176 * @returns VBox strict status code.
5177 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5178 * @param cbInstr The number of bytes to offset rIP by in the return
5179 * address.
5180 * @param u8Vector The interrupt / exception vector number.
5181 * @param fFlags The flags.
5182 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
5183 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
5184 */
5185IEM_STATIC VBOXSTRICTRC
5186iemRaiseXcptOrIntInLongMode(PVMCPU pVCpu,
5187 uint8_t cbInstr,
5188 uint8_t u8Vector,
5189 uint32_t fFlags,
5190 uint16_t uErr,
5191 uint64_t uCr2)
5192{
5193 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5194
5195 /*
5196 * Read the IDT entry.
5197 */
5198 uint16_t offIdt = (uint16_t)u8Vector << 4;
5199 if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
5200 {
5201 Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
5202 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5203 }
5204 X86DESC64 Idte;
5205 VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
5206 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5207 rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
5208 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
5209 {
5210 Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
5211 return rcStrict;
5212 }
5213 Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
5214 u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
5215 Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
5216
5217 /*
5218 * Check the descriptor type, DPL and such.
5219 * ASSUMES this is done in the same order as described for call-gate calls.
5220 */
5221 if (Idte.Gate.u1DescType)
5222 {
5223 Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
5224 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5225 }
5226 uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
5227 switch (Idte.Gate.u4Type)
5228 {
5229 case AMD64_SEL_TYPE_SYS_INT_GATE:
5230 fEflToClear |= X86_EFL_IF;
5231 break;
5232 case AMD64_SEL_TYPE_SYS_TRAP_GATE:
5233 break;
5234
5235 default:
5236 Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
5237 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5238 }
5239
5240 /* Check DPL against CPL if applicable. */
5241 if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
5242 {
5243 if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
5244 {
5245 Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
5246 return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5247 }
5248 }
5249
5250 /* Is it there? */
5251 if (!Idte.Gate.u1Present)
5252 {
5253 Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
5254 return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
5255 }
5256
5257 /* A null CS is bad. */
5258 RTSEL NewCS = Idte.Gate.u16Sel;
5259 if (!(NewCS & X86_SEL_MASK_OFF_RPL))
5260 {
5261 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
5262 return iemRaiseGeneralProtectionFault0(pVCpu);
5263 }
5264
5265 /* Fetch the descriptor for the new CS. */
5266 IEMSELDESC DescCS;
5267 rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
5268 if (rcStrict != VINF_SUCCESS)
5269 {
5270 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
5271 return rcStrict;
5272 }
5273
5274 /* Must be a 64-bit code segment. */
5275 if (!DescCS.Long.Gen.u1DescType)
5276 {
5277 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
5278 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5279 }
5280 if ( !DescCS.Long.Gen.u1Long
5281 || DescCS.Long.Gen.u1DefBig
5282 || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
5283 {
5284 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
5285 u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
5286 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5287 }
5288
5289 /* Don't allow lowering the privilege level. For non-conforming CS
5290 selectors, the CS.DPL sets the privilege level the trap/interrupt
5291 handler runs at. For conforming CS selectors, the CPL remains
5292 unchanged, but the CS.DPL must be <= CPL. */
5293 /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
5294 * when CPU in Ring-0. Result \#GP? */
5295 if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
5296 {
5297 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
5298 u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
5299 return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
5300 }
5301
5302
5303 /* Make sure the selector is present. */
5304 if (!DescCS.Legacy.Gen.u1Present)
5305 {
5306 Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
5307 return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
5308 }
5309
5310 /* Check that the new RIP is canonical. */
5311 uint64_t const uNewRip = Idte.Gate.u16OffsetLow
5312 | ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
5313 | ((uint64_t)Idte.Gate.u32OffsetTop << 32);
5314 if (!IEM_IS_CANONICAL(uNewRip))
5315 {
5316 Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
5317 return iemRaiseGeneralProtectionFault0(pVCpu);
5318 }
5319
5320 /*
5321 * If the privilege level changes or if the IST isn't zero, we need to get
5322 * a new stack from the TSS.
5323 */
5324 uint64_t uNewRsp;
5325 uint8_t const uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
5326 ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
5327 if ( uNewCpl != pVCpu->iem.s.uCpl
5328 || Idte.Gate.u3IST != 0)
5329 {
5330 rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
5331 if (rcStrict != VINF_SUCCESS)
5332 return rcStrict;
5333 }
5334 else
5335 uNewRsp = pVCpu->cpum.GstCtx.rsp;
5336 uNewRsp &= ~(uint64_t)0xf;
5337
5338 /*
5339 * Calc the flag image to push.
5340 */
5341 uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
5342 if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
5343 fEfl &= ~X86_EFL_RF;
5344 else
5345 fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
5346
5347 /*
5348 * Start making changes.
5349 */
5350 /* Set the new CPL so that stack accesses use it. */
5351 uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
5352 pVCpu->iem.s.uCpl = uNewCpl;
5353
5354 /* Create the stack frame. */
5355 uint32_t cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
5356 RTPTRUNION uStackFrame;
5357 rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
5358 uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS); /* _SYS is a hack ... */
5359 if (rcStrict != VINF_SUCCESS)
5360 return rcStrict;
5361 void * const pvStackFrame = uStackFrame.pv;
5362
5363 if (fFlags & IEM_XCPT_FLAGS_ERR)
5364 *uStackFrame.pu64++ = uErr;
5365 uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
5366 uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */
5367 uStackFrame.pu64[2] = fEfl;
5368 uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
5369 uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
5370 rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
5371 if (rcStrict != VINF_SUCCESS)
5372 return rcStrict;
5373
5374 /* Mark the CS selectors 'accessed' (hope this is the correct time). */
5375 /** @todo testcase: excatly _when_ are the accessed bits set - before or
5376 * after pushing the stack frame? (Write protect the gdt + stack to
5377 * find out.) */
5378 if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
5379 {
5380 rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
5381 if (rcStrict != VINF_SUCCESS)
5382 return rcStrict;
5383 DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
5384 }
5385
5386 /*
5387 * Start comitting the register changes.
5388 */
5389 /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
5390 * hidden registers when interrupting 32-bit or 16-bit code! */
5391 if (uNewCpl != uOldCpl)
5392 {
5393 pVCpu->cpum.GstCtx.ss.Sel = 0 | uNewCpl;
5394 pVCpu->cpum.GstCtx.ss.ValidSel = 0 | uNewCpl;
5395 pVCpu->cpum.GstCtx.ss.fFlags = CPUMSELREG_FLAGS_VALID;
5396 pVCpu->cpum.GstCtx.ss.u32Limit = UINT32_MAX;
5397 pVCpu->cpum.GstCtx.ss.u64Base = 0;
5398 pVCpu->cpum.GstCtx.ss.Attr.u = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE;
5399 }
5400 pVCpu->cpum.GstCtx.rsp = uNewRsp - cbStackFrame;
5401 pVCpu->cpum.GstCtx.cs.Sel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5402 pVCpu->cpum.GstCtx.cs.ValidSel = (NewCS & ~X86_SEL_RPL) | uNewCpl;
5403 pVCpu->cpum.GstCtx.cs.fFlags = CPUMSELREG_FLAGS_VALID;
5404 pVCpu->cpum.GstCtx.cs.u32Limit = X86DESC_LIMIT_G(&DescCS.Legacy);
5405 pVCpu->cpum.GstCtx.cs.u64Base = X86DESC_BASE(&DescCS.Legacy);
5406 pVCpu->cpum.GstCtx.cs.Attr.u = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
5407 pVCpu->cpum.GstCtx.rip = uNewRip;
5408
5409 fEfl &= ~fEflToClear;
5410 IEMMISC_SET_EFL(pVCpu, fEfl);
5411
5412 if (fFlags & IEM_XCPT_FLAGS_CR2)
5413 pVCpu->cpum.GstCtx.cr2 = uCr2;
5414
5415 if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
5416 iemRaiseXcptAdjustState(pVCpu, u8Vector);
5417
5418 return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
5419}
5420
5421
5422/**
5423 * Implements exceptions and interrupts.
5424 *
5425 * All exceptions and interrupts goes thru this function!
5426 *
5427 * @returns VBox strict status code.
5428 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5429 * @param cbInstr The number of bytes to offset rIP by in the return
5430 * address.
5431 * @param u8Vector The interrupt / exception vector number.
5432 * @param fFlags The flags.
5433 * @param uErr The error value if IEM_XCPT_FLAGS_ERR is set.
5434 * @param uCr2 The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
5435 */
5436DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC)
5437iemRaiseXcptOrInt(PVMCPU pVCpu,
5438 uint8_t cbInstr,
5439 uint8_t u8Vector,
5440 uint32_t fFlags,
5441 uint16_t uErr,
5442 uint64_t uCr2)
5443{
5444 /*
5445 * Get all the state that we might need here.
5446 */
5447 IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5448 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
5449
5450#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
5451 /*
5452 * Flush prefetch buffer
5453 */
5454 pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
5455#endif
5456
5457 /*
5458 * Perform the V8086 IOPL check and upgrade the fault without nesting.
5459 */
5460 if ( pVCpu->cpum.GstCtx.eflags.Bits.u1VM
5461 && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
5462 && (fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_BP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
5463 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
5464 {
5465 Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
5466 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
5467 u8Vector = X86_XCPT_GP;
5468 uErr = 0;
5469 }
5470#ifdef DBGFTRACE_ENABLED
5471 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
5472 pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
5473 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
5474#endif
5475
5476#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5477 if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
5478 {
5479 /*
5480 * If the event is being injected as part of VMRUN, it isn't subject to event
5481 * intercepts in the nested-guest. However, secondary exceptions that occur
5482 * during injection of any event -are- subject to exception intercepts.
5483 * See AMD spec. 15.20 "Event Injection".
5484 */
5485 if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
5486 pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = 1;
5487 else
5488 {
5489 /*
5490 * Check and handle if the event being raised is intercepted.
5491 */
5492 VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
5493 if (rcStrict0 != VINF_HM_INTERCEPT_NOT_ACTIVE)
5494 return rcStrict0;
5495 }
5496 }
5497#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5498
5499 /*
5500 * Do recursion accounting.
5501 */
5502 uint8_t const uPrevXcpt = pVCpu->iem.s.uCurXcpt;
5503 uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
5504 if (pVCpu->iem.s.cXcptRecursions == 0)
5505 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
5506 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
5507 else
5508 {
5509 Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
5510 u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
5511 pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
5512
5513 if (pVCpu->iem.s.cXcptRecursions >= 3)
5514 {
5515#ifdef DEBUG_bird
5516 AssertFailed();
5517#endif
5518 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
5519 }
5520
5521 /*
5522 * Evaluate the sequence of recurring events.
5523 */
5524 IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
5525 NULL /* pXcptRaiseInfo */);
5526 if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
5527 { /* likely */ }
5528 else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
5529 {
5530 Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
5531 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
5532 u8Vector = X86_XCPT_DF;
5533 uErr = 0;
5534 /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
5535 if (IEM_IS_SVM_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
5536 IEM_RETURN_SVM_VMEXIT(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
5537 }
5538 else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
5539 {
5540 Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
5541 return iemInitiateCpuShutdown(pVCpu);
5542 }
5543 else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
5544 {
5545 /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
5546 Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
5547 if ( !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
5548 && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
5549 return VERR_EM_GUEST_CPU_HANG;
5550 }
5551 else
5552 {
5553 AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
5554 enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
5555 return VERR_IEM_IPE_9;
5556 }
5557
5558 /*
5559 * The 'EXT' bit is set when an exception occurs during deliver of an external
5560 * event (such as an interrupt or earlier exception)[1]. Privileged software
5561 * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
5562 * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
5563 *
5564 * [1] - Intel spec. 6.13 "Error Code"
5565 * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
5566 * [3] - Intel Instruction reference for INT n.
5567 */
5568 if ( (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR))
5569 && (fFlags & IEM_XCPT_FLAGS_ERR)
5570 && u8Vector != X86_XCPT_PF
5571 && u8Vector != X86_XCPT_DF)
5572 {
5573 uErr |= X86_TRAP_ERR_EXTERNAL;
5574 }
5575 }
5576
5577 pVCpu->iem.s.cXcptRecursions++;
5578 pVCpu->iem.s.uCurXcpt = u8Vector;
5579 pVCpu->iem.s.fCurXcpt = fFlags;
5580 pVCpu->iem.s.uCurXcptErr = uErr;
5581 pVCpu->iem.s.uCurXcptCr2 = uCr2;
5582
5583 /*
5584 * Extensive logging.
5585 */
5586#if defined(LOG_ENABLED) && defined(IN_RING3)
5587 if (LogIs3Enabled())
5588 {
5589 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
5590 PVM pVM = pVCpu->CTX_SUFF(pVM);
5591 char szRegs[4096];
5592 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
5593 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
5594 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
5595 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
5596 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
5597 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
5598 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
5599 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
5600 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
5601 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
5602 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
5603 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
5604 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
5605 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
5606 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
5607 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
5608 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
5609 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
5610 " efer=%016VR{efer}\n"
5611 " pat=%016VR{pat}\n"
5612 " sf_mask=%016VR{sf_mask}\n"
5613 "krnl_gs_base=%016VR{krnl_gs_base}\n"
5614 " lstar=%016VR{lstar}\n"
5615 " star=%016VR{star} cstar=%016VR{cstar}\n"
5616 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
5617 );
5618
5619 char szInstr[256];
5620 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
5621 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
5622 szInstr, sizeof(szInstr), NULL);
5623 Log3(("%s%s\n", szRegs, szInstr));
5624 }
5625#endif /* LOG_ENABLED */
5626
5627 /*
5628 * Call the mode specific worker function.
5629 */
5630 VBOXSTRICTRC rcStrict;
5631 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
5632 rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5633 else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
5634 rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5635 else
5636 rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5637
5638 /* Flush the prefetch buffer. */
5639#ifdef IEM_WITH_CODE_TLB
5640 pVCpu->iem.s.pbInstrBuf = NULL;
5641#else
5642 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
5643#endif
5644
5645 /*
5646 * Unwind.
5647 */
5648 pVCpu->iem.s.cXcptRecursions--;
5649 pVCpu->iem.s.uCurXcpt = uPrevXcpt;
5650 pVCpu->iem.s.fCurXcpt = fPrevXcpt;
5651 Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
5652 VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
5653 pVCpu->iem.s.cXcptRecursions + 1));
5654 return rcStrict;
5655}
5656
5657#ifdef IEM_WITH_SETJMP
5658/**
5659 * See iemRaiseXcptOrInt. Will not return.
5660 */
5661IEM_STATIC DECL_NO_RETURN(void)
5662iemRaiseXcptOrIntJmp(PVMCPU pVCpu,
5663 uint8_t cbInstr,
5664 uint8_t u8Vector,
5665 uint32_t fFlags,
5666 uint16_t uErr,
5667 uint64_t uCr2)
5668{
5669 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
5670 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
5671}
5672#endif
5673
5674
5675/** \#DE - 00. */
5676DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDivideError(PVMCPU pVCpu)
5677{
5678 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5679}
5680
5681
5682/** \#DB - 01.
5683 * @note This automatically clear DR7.GD. */
5684DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDebugException(PVMCPU pVCpu)
5685{
5686 /** @todo set/clear RF. */
5687 pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
5688 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5689}
5690
5691
5692/** \#BR - 05. */
5693DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseBoundRangeExceeded(PVMCPU pVCpu)
5694{
5695 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5696}
5697
5698
5699/** \#UD - 06. */
5700DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseUndefinedOpcode(PVMCPU pVCpu)
5701{
5702 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5703}
5704
5705
5706/** \#NM - 07. */
5707DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseDeviceNotAvailable(PVMCPU pVCpu)
5708{
5709 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5710}
5711
5712
5713/** \#TS(err) - 0a. */
5714DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultWithErr(PVMCPU pVCpu, uint16_t uErr)
5715{
5716 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5717}
5718
5719
5720/** \#TS(tr) - 0a. */
5721DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultCurrentTSS(PVMCPU pVCpu)
5722{
5723 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5724 pVCpu->cpum.GstCtx.tr.Sel, 0);
5725}
5726
5727
5728/** \#TS(0) - 0a. */
5729DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFault0(PVMCPU pVCpu)
5730{
5731 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5732 0, 0);
5733}
5734
5735
5736/** \#TS(err) - 0a. */
5737DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseTaskSwitchFaultBySelector(PVMCPU pVCpu, uint16_t uSel)
5738{
5739 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5740 uSel & X86_SEL_MASK_OFF_RPL, 0);
5741}
5742
5743
5744/** \#NP(err) - 0b. */
5745DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentWithErr(PVMCPU pVCpu, uint16_t uErr)
5746{
5747 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5748}
5749
5750
5751/** \#NP(sel) - 0b. */
5752DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorNotPresentBySelector(PVMCPU pVCpu, uint16_t uSel)
5753{
5754 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5755 uSel & ~X86_SEL_RPL, 0);
5756}
5757
5758
5759/** \#SS(seg) - 0c. */
5760DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentBySelector(PVMCPU pVCpu, uint16_t uSel)
5761{
5762 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5763 uSel & ~X86_SEL_RPL, 0);
5764}
5765
5766
5767/** \#SS(err) - 0c. */
5768DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseStackSelectorNotPresentWithErr(PVMCPU pVCpu, uint16_t uErr)
5769{
5770 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5771}
5772
5773
5774/** \#GP(n) - 0d. */
5775DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault(PVMCPU pVCpu, uint16_t uErr)
5776{
5777 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
5778}
5779
5780
5781/** \#GP(0) - 0d. */
5782DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFault0(PVMCPU pVCpu)
5783{
5784 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5785}
5786
5787#ifdef IEM_WITH_SETJMP
5788/** \#GP(0) - 0d. */
5789DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseGeneralProtectionFault0Jmp(PVMCPU pVCpu)
5790{
5791 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5792}
5793#endif
5794
5795
5796/** \#GP(sel) - 0d. */
5797DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseGeneralProtectionFaultBySelector(PVMCPU pVCpu, RTSEL Sel)
5798{
5799 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
5800 Sel & ~X86_SEL_RPL, 0);
5801}
5802
5803
5804/** \#GP(0) - 0d. */
5805DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseNotCanonical(PVMCPU pVCpu)
5806{
5807 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5808}
5809
5810
5811/** \#GP(sel) - 0d. */
5812DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBounds(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess)
5813{
5814 NOREF(iSegReg); NOREF(fAccess);
5815 return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
5816 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5817}
5818
5819#ifdef IEM_WITH_SETJMP
5820/** \#GP(sel) - 0d, longjmp. */
5821DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsJmp(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess)
5822{
5823 NOREF(iSegReg); NOREF(fAccess);
5824 iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
5825 IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5826}
5827#endif
5828
5829/** \#GP(sel) - 0d. */
5830DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorBoundsBySelector(PVMCPU pVCpu, RTSEL Sel)
5831{
5832 NOREF(Sel);
5833 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5834}
5835
5836#ifdef IEM_WITH_SETJMP
5837/** \#GP(sel) - 0d, longjmp. */
5838DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorBoundsBySelectorJmp(PVMCPU pVCpu, RTSEL Sel)
5839{
5840 NOREF(Sel);
5841 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5842}
5843#endif
5844
5845
5846/** \#GP(sel) - 0d. */
5847DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseSelectorInvalidAccess(PVMCPU pVCpu, uint32_t iSegReg, uint32_t fAccess)
5848{
5849 NOREF(iSegReg); NOREF(fAccess);
5850 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5851}
5852
5853#ifdef IEM_WITH_SETJMP
5854/** \#GP(sel) - 0d, longjmp. */
5855DECL_NO_INLINE(IEM_STATIC, DECL_NO_RETURN(void)) iemRaiseSelectorInvalidAccessJmp(PVMCPU pVCpu, uint32_t iSegReg,
5856 uint32_t fAccess)
5857{
5858 NOREF(iSegReg); NOREF(fAccess);
5859 iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
5860}
5861#endif
5862
5863
5864/** \#PF(n) - 0e. */
5865DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaisePageFault(PVMCPU pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
5866{
5867 uint16_t uErr;
5868 switch (rc)
5869 {
5870 case VERR_PAGE_NOT_PRESENT:
5871 case VERR_PAGE_TABLE_NOT_PRESENT:
5872 case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
5873 case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
5874 uErr = 0;
5875 break;
5876
5877 default:
5878 AssertMsgFailed(("%Rrc\n", rc));
5879 RT_FALL_THRU();
5880 case VERR_ACCESS_DENIED:
5881 uErr = X86_TRAP_PF_P;
5882 break;
5883
5884 /** @todo reserved */
5885 }
5886
5887 if (pVCpu->iem.s.uCpl == 3)
5888 uErr |= X86_TRAP_PF_US;
5889
5890 if ( (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
5891 && ( (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
5892 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
5893 uErr |= X86_TRAP_PF_ID;
5894
5895#if 0 /* This is so much non-sense, really. Why was it done like that? */
5896 /* Note! RW access callers reporting a WRITE protection fault, will clear
5897 the READ flag before calling. So, read-modify-write accesses (RW)
5898 can safely be reported as READ faults. */
5899 if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
5900 uErr |= X86_TRAP_PF_RW;
5901#else
5902 if (fAccess & IEM_ACCESS_TYPE_WRITE)
5903 {
5904 if (!(fAccess & IEM_ACCESS_TYPE_READ))
5905 uErr |= X86_TRAP_PF_RW;
5906 }
5907#endif
5908
5909 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2,
5910 uErr, GCPtrWhere);
5911}
5912
5913#ifdef IEM_WITH_SETJMP
5914/** \#PF(n) - 0e, longjmp. */
5915IEM_STATIC DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPU pVCpu, RTGCPTR GCPtrWhere, uint32_t fAccess, int rc)
5916{
5917 longjmp(*CTX_SUFF(pVCpu->iem.s.pJmpBuf), VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, fAccess, rc)));
5918}
5919#endif
5920
5921
5922/** \#MF(0) - 10. */
5923DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseMathFault(PVMCPU pVCpu)
5924{
5925 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5926}
5927
5928
5929/** \#AC(0) - 11. */
5930DECL_NO_INLINE(IEM_STATIC, VBOXSTRICTRC) iemRaiseAlignmentCheckException(PVMCPU pVCpu)
5931{
5932 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5933}
5934
5935
5936/**
5937 * Macro for calling iemCImplRaiseDivideError().
5938 *
5939 * This enables us to add/remove arguments and force different levels of
5940 * inlining as we wish.
5941 *
5942 * @return Strict VBox status code.
5943 */
5944#define IEMOP_RAISE_DIVIDE_ERROR() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseDivideError)
5945IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
5946{
5947 NOREF(cbInstr);
5948 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5949}
5950
5951
5952/**
5953 * Macro for calling iemCImplRaiseInvalidLockPrefix().
5954 *
5955 * This enables us to add/remove arguments and force different levels of
5956 * inlining as we wish.
5957 *
5958 * @return Strict VBox status code.
5959 */
5960#define IEMOP_RAISE_INVALID_LOCK_PREFIX() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidLockPrefix)
5961IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
5962{
5963 NOREF(cbInstr);
5964 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5965}
5966
5967
5968/**
5969 * Macro for calling iemCImplRaiseInvalidOpcode().
5970 *
5971 * This enables us to add/remove arguments and force different levels of
5972 * inlining as we wish.
5973 *
5974 * @return Strict VBox status code.
5975 */
5976#define IEMOP_RAISE_INVALID_OPCODE() IEM_MC_DEFER_TO_CIMPL_0(iemCImplRaiseInvalidOpcode)
5977IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
5978{
5979 NOREF(cbInstr);
5980 return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
5981}
5982
5983
5984/** @} */
5985
5986
5987/*
5988 *
5989 * Helpers routines.
5990 * Helpers routines.
5991 * Helpers routines.
5992 *
5993 */
5994
5995/**
5996 * Recalculates the effective operand size.
5997 *
5998 * @param pVCpu The cross context virtual CPU structure of the calling thread.
5999 */
6000IEM_STATIC void iemRecalEffOpSize(PVMCPU pVCpu)
6001{
6002 switch (pVCpu->iem.s.enmCpuMode)
6003 {
6004 case IEMMODE_16BIT:
6005 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
6006 break;
6007 case IEMMODE_32BIT:
6008 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
6009 break;
6010 case IEMMODE_64BIT:
6011 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
6012 {
6013 case 0:
6014 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
6015 break;
6016 case IEM_OP_PRF_SIZE_OP:
6017 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
6018 break;
6019 case IEM_OP_PRF_SIZE_REX_W:
6020 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
6021 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
6022 break;
6023 }
6024 break;
6025 default:
6026 AssertFailed();
6027 }
6028}
6029
6030
6031/**
6032 * Sets the default operand size to 64-bit and recalculates the effective
6033 * operand size.
6034 *
6035 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6036 */
6037IEM_STATIC void iemRecalEffOpSize64Default(PVMCPU pVCpu)
6038{
6039 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6040 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
6041 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
6042 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
6043 else
6044 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
6045}
6046
6047
6048/*
6049 *
6050 * Common opcode decoders.
6051 * Common opcode decoders.
6052 * Common opcode decoders.
6053 *
6054 */
6055//#include <iprt/mem.h>
6056
6057/**
6058 * Used to add extra details about a stub case.
6059 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6060 */
6061IEM_STATIC void iemOpStubMsg2(PVMCPU pVCpu)
6062{
6063#if defined(LOG_ENABLED) && defined(IN_RING3)
6064 PVM pVM = pVCpu->CTX_SUFF(pVM);
6065 char szRegs[4096];
6066 DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
6067 "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
6068 "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
6069 "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
6070 "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
6071 "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
6072 "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
6073 "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
6074 "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
6075 "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
6076 "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
6077 "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
6078 "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
6079 "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
6080 "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
6081 "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
6082 "tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
6083 " sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
6084 " efer=%016VR{efer}\n"
6085 " pat=%016VR{pat}\n"
6086 " sf_mask=%016VR{sf_mask}\n"
6087 "krnl_gs_base=%016VR{krnl_gs_base}\n"
6088 " lstar=%016VR{lstar}\n"
6089 " star=%016VR{star} cstar=%016VR{cstar}\n"
6090 "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
6091 );
6092
6093 char szInstr[256];
6094 DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
6095 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
6096 szInstr, sizeof(szInstr), NULL);
6097
6098 RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
6099#else
6100 RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
6101#endif
6102}
6103
6104/**
6105 * Complains about a stub.
6106 *
6107 * Providing two versions of this macro, one for daily use and one for use when
6108 * working on IEM.
6109 */
6110#if 0
6111# define IEMOP_BITCH_ABOUT_STUB() \
6112 do { \
6113 RTAssertMsg1(NULL, __LINE__, __FILE__, __FUNCTION__); \
6114 iemOpStubMsg2(pVCpu); \
6115 RTAssertPanic(); \
6116 } while (0)
6117#else
6118# define IEMOP_BITCH_ABOUT_STUB() Log(("Stub: %s (line %d)\n", __FUNCTION__, __LINE__));
6119#endif
6120
6121/** Stubs an opcode. */
6122#define FNIEMOP_STUB(a_Name) \
6123 FNIEMOP_DEF(a_Name) \
6124 { \
6125 RT_NOREF_PV(pVCpu); \
6126 IEMOP_BITCH_ABOUT_STUB(); \
6127 return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
6128 } \
6129 typedef int ignore_semicolon
6130
6131/** Stubs an opcode. */
6132#define FNIEMOP_STUB_1(a_Name, a_Type0, a_Name0) \
6133 FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
6134 { \
6135 RT_NOREF_PV(pVCpu); \
6136 RT_NOREF_PV(a_Name0); \
6137 IEMOP_BITCH_ABOUT_STUB(); \
6138 return VERR_IEM_INSTR_NOT_IMPLEMENTED; \
6139 } \
6140 typedef int ignore_semicolon
6141
6142/** Stubs an opcode which currently should raise \#UD. */
6143#define FNIEMOP_UD_STUB(a_Name) \
6144 FNIEMOP_DEF(a_Name) \
6145 { \
6146 Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
6147 return IEMOP_RAISE_INVALID_OPCODE(); \
6148 } \
6149 typedef int ignore_semicolon
6150
6151/** Stubs an opcode which currently should raise \#UD. */
6152#define FNIEMOP_UD_STUB_1(a_Name, a_Type0, a_Name0) \
6153 FNIEMOP_DEF_1(a_Name, a_Type0, a_Name0) \
6154 { \
6155 RT_NOREF_PV(pVCpu); \
6156 RT_NOREF_PV(a_Name0); \
6157 Log(("Unsupported instruction %Rfn\n", __FUNCTION__)); \
6158 return IEMOP_RAISE_INVALID_OPCODE(); \
6159 } \
6160 typedef int ignore_semicolon
6161
6162
6163
6164/** @name Register Access.
6165 * @{
6166 */
6167
6168/**
6169 * Gets a reference (pointer) to the specified hidden segment register.
6170 *
6171 * @returns Hidden register reference.
6172 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6173 * @param iSegReg The segment register.
6174 */
6175IEM_STATIC PCPUMSELREG iemSRegGetHid(PVMCPU pVCpu, uint8_t iSegReg)
6176{
6177 Assert(iSegReg < X86_SREG_COUNT);
6178 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6179 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6180
6181#ifdef VBOX_WITH_RAW_MODE_NOT_R0
6182 if (RT_LIKELY(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg)))
6183 { /* likely */ }
6184 else
6185 CPUMGuestLazyLoadHiddenSelectorReg(pVCpu, pSReg);
6186#else
6187 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
6188#endif
6189 return pSReg;
6190}
6191
6192
6193/**
6194 * Ensures that the given hidden segment register is up to date.
6195 *
6196 * @returns Hidden register reference.
6197 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6198 * @param pSReg The segment register.
6199 */
6200IEM_STATIC PCPUMSELREG iemSRegUpdateHid(PVMCPU pVCpu, PCPUMSELREG pSReg)
6201{
6202#ifdef VBOX_WITH_RAW_MODE_NOT_R0
6203 if (!CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
6204 CPUMGuestLazyLoadHiddenSelectorReg(pVCpu, pSReg);
6205#else
6206 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
6207 NOREF(pVCpu);
6208#endif
6209 return pSReg;
6210}
6211
6212
6213/**
6214 * Gets a reference (pointer) to the specified segment register (the selector
6215 * value).
6216 *
6217 * @returns Pointer to the selector variable.
6218 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6219 * @param iSegReg The segment register.
6220 */
6221DECLINLINE(uint16_t *) iemSRegRef(PVMCPU pVCpu, uint8_t iSegReg)
6222{
6223 Assert(iSegReg < X86_SREG_COUNT);
6224 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6225 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
6226}
6227
6228
6229/**
6230 * Fetches the selector value of a segment register.
6231 *
6232 * @returns The selector value.
6233 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6234 * @param iSegReg The segment register.
6235 */
6236DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPU pVCpu, uint8_t iSegReg)
6237{
6238 Assert(iSegReg < X86_SREG_COUNT);
6239 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6240 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
6241}
6242
6243
6244/**
6245 * Fetches the base address value of a segment register.
6246 *
6247 * @returns The selector value.
6248 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6249 * @param iSegReg The segment register.
6250 */
6251DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPU pVCpu, uint8_t iSegReg)
6252{
6253 Assert(iSegReg < X86_SREG_COUNT);
6254 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6255 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6256}
6257
6258
6259/**
6260 * Gets a reference (pointer) to the specified general purpose register.
6261 *
6262 * @returns Register reference.
6263 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6264 * @param iReg The general purpose register.
6265 */
6266DECLINLINE(void *) iemGRegRef(PVMCPU pVCpu, uint8_t iReg)
6267{
6268 Assert(iReg < 16);
6269 return &pVCpu->cpum.GstCtx.aGRegs[iReg];
6270}
6271
6272
6273/**
6274 * Gets a reference (pointer) to the specified 8-bit general purpose register.
6275 *
6276 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
6277 *
6278 * @returns Register reference.
6279 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6280 * @param iReg The register.
6281 */
6282DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPU pVCpu, uint8_t iReg)
6283{
6284 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
6285 {
6286 Assert(iReg < 16);
6287 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
6288 }
6289 /* high 8-bit register. */
6290 Assert(iReg < 8);
6291 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
6292}
6293
6294
6295/**
6296 * Gets a reference (pointer) to the specified 16-bit general purpose register.
6297 *
6298 * @returns Register reference.
6299 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6300 * @param iReg The register.
6301 */
6302DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPU pVCpu, uint8_t iReg)
6303{
6304 Assert(iReg < 16);
6305 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
6306}
6307
6308
6309/**
6310 * Gets a reference (pointer) to the specified 32-bit general purpose register.
6311 *
6312 * @returns Register reference.
6313 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6314 * @param iReg The register.
6315 */
6316DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPU pVCpu, uint8_t iReg)
6317{
6318 Assert(iReg < 16);
6319 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
6320}
6321
6322
6323/**
6324 * Gets a reference (pointer) to the specified 64-bit general purpose register.
6325 *
6326 * @returns Register reference.
6327 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6328 * @param iReg The register.
6329 */
6330DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPU pVCpu, uint8_t iReg)
6331{
6332 Assert(iReg < 64);
6333 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
6334}
6335
6336
6337/**
6338 * Gets a reference (pointer) to the specified segment register's base address.
6339 *
6340 * @returns Segment register base address reference.
6341 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6342 * @param iSegReg The segment selector.
6343 */
6344DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPU pVCpu, uint8_t iSegReg)
6345{
6346 Assert(iSegReg < X86_SREG_COUNT);
6347 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
6348 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6349}
6350
6351
6352/**
6353 * Fetches the value of a 8-bit general purpose register.
6354 *
6355 * @returns The register value.
6356 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6357 * @param iReg The register.
6358 */
6359DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPU pVCpu, uint8_t iReg)
6360{
6361 return *iemGRegRefU8(pVCpu, iReg);
6362}
6363
6364
6365/**
6366 * Fetches the value of a 16-bit general purpose register.
6367 *
6368 * @returns The register value.
6369 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6370 * @param iReg The register.
6371 */
6372DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPU pVCpu, uint8_t iReg)
6373{
6374 Assert(iReg < 16);
6375 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
6376}
6377
6378
6379/**
6380 * Fetches the value of a 32-bit general purpose register.
6381 *
6382 * @returns The register value.
6383 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6384 * @param iReg The register.
6385 */
6386DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPU pVCpu, uint8_t iReg)
6387{
6388 Assert(iReg < 16);
6389 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
6390}
6391
6392
6393/**
6394 * Fetches the value of a 64-bit general purpose register.
6395 *
6396 * @returns The register value.
6397 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6398 * @param iReg The register.
6399 */
6400DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPU pVCpu, uint8_t iReg)
6401{
6402 Assert(iReg < 16);
6403 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
6404}
6405
6406
6407/**
6408 * Adds a 8-bit signed jump offset to RIP/EIP/IP.
6409 *
6410 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6411 * segment limit.
6412 *
6413 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6414 * @param offNextInstr The offset of the next instruction.
6415 */
6416IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS8(PVMCPU pVCpu, int8_t offNextInstr)
6417{
6418 switch (pVCpu->iem.s.enmEffOpSize)
6419 {
6420 case IEMMODE_16BIT:
6421 {
6422 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6423 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
6424 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6425 return iemRaiseGeneralProtectionFault0(pVCpu);
6426 pVCpu->cpum.GstCtx.rip = uNewIp;
6427 break;
6428 }
6429
6430 case IEMMODE_32BIT:
6431 {
6432 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6433 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6434
6435 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6436 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
6437 return iemRaiseGeneralProtectionFault0(pVCpu);
6438 pVCpu->cpum.GstCtx.rip = uNewEip;
6439 break;
6440 }
6441
6442 case IEMMODE_64BIT:
6443 {
6444 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6445
6446 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6447 if (!IEM_IS_CANONICAL(uNewRip))
6448 return iemRaiseGeneralProtectionFault0(pVCpu);
6449 pVCpu->cpum.GstCtx.rip = uNewRip;
6450 break;
6451 }
6452
6453 IEM_NOT_REACHED_DEFAULT_CASE_RET();
6454 }
6455
6456 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6457
6458#ifndef IEM_WITH_CODE_TLB
6459 /* Flush the prefetch buffer. */
6460 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6461#endif
6462
6463 return VINF_SUCCESS;
6464}
6465
6466
6467/**
6468 * Adds a 16-bit signed jump offset to RIP/EIP/IP.
6469 *
6470 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6471 * segment limit.
6472 *
6473 * @returns Strict VBox status code.
6474 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6475 * @param offNextInstr The offset of the next instruction.
6476 */
6477IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS16(PVMCPU pVCpu, int16_t offNextInstr)
6478{
6479 Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
6480
6481 uint16_t uNewIp = pVCpu->cpum.GstCtx.ip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6482 if ( uNewIp > pVCpu->cpum.GstCtx.cs.u32Limit
6483 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6484 return iemRaiseGeneralProtectionFault0(pVCpu);
6485 /** @todo Test 16-bit jump in 64-bit mode. possible? */
6486 pVCpu->cpum.GstCtx.rip = uNewIp;
6487 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6488
6489#ifndef IEM_WITH_CODE_TLB
6490 /* Flush the prefetch buffer. */
6491 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6492#endif
6493
6494 return VINF_SUCCESS;
6495}
6496
6497
6498/**
6499 * Adds a 32-bit signed jump offset to RIP/EIP/IP.
6500 *
6501 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6502 * segment limit.
6503 *
6504 * @returns Strict VBox status code.
6505 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6506 * @param offNextInstr The offset of the next instruction.
6507 */
6508IEM_STATIC VBOXSTRICTRC iemRegRipRelativeJumpS32(PVMCPU pVCpu, int32_t offNextInstr)
6509{
6510 Assert(pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT);
6511
6512 if (pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT)
6513 {
6514 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6515
6516 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6517 if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
6518 return iemRaiseGeneralProtectionFault0(pVCpu);
6519 pVCpu->cpum.GstCtx.rip = uNewEip;
6520 }
6521 else
6522 {
6523 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6524
6525 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + offNextInstr + IEM_GET_INSTR_LEN(pVCpu);
6526 if (!IEM_IS_CANONICAL(uNewRip))
6527 return iemRaiseGeneralProtectionFault0(pVCpu);
6528 pVCpu->cpum.GstCtx.rip = uNewRip;
6529 }
6530 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6531
6532#ifndef IEM_WITH_CODE_TLB
6533 /* Flush the prefetch buffer. */
6534 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6535#endif
6536
6537 return VINF_SUCCESS;
6538}
6539
6540
6541/**
6542 * Performs a near jump to the specified address.
6543 *
6544 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
6545 * segment limit.
6546 *
6547 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6548 * @param uNewRip The new RIP value.
6549 */
6550IEM_STATIC VBOXSTRICTRC iemRegRipJump(PVMCPU pVCpu, uint64_t uNewRip)
6551{
6552 switch (pVCpu->iem.s.enmEffOpSize)
6553 {
6554 case IEMMODE_16BIT:
6555 {
6556 Assert(uNewRip <= UINT16_MAX);
6557 if ( uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit
6558 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) /* no need to check for non-canonical. */
6559 return iemRaiseGeneralProtectionFault0(pVCpu);
6560 /** @todo Test 16-bit jump in 64-bit mode. */
6561 pVCpu->cpum.GstCtx.rip = uNewRip;
6562 break;
6563 }
6564
6565 case IEMMODE_32BIT:
6566 {
6567 Assert(uNewRip <= UINT32_MAX);
6568 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6569 Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
6570
6571 if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
6572 return iemRaiseGeneralProtectionFault0(pVCpu);
6573 pVCpu->cpum.GstCtx.rip = uNewRip;
6574 break;
6575 }
6576
6577 case IEMMODE_64BIT:
6578 {
6579 Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
6580
6581 if (!IEM_IS_CANONICAL(uNewRip))
6582 return iemRaiseGeneralProtectionFault0(pVCpu);
6583 pVCpu->cpum.GstCtx.rip = uNewRip;
6584 break;
6585 }
6586
6587 IEM_NOT_REACHED_DEFAULT_CASE_RET();
6588 }
6589
6590 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6591
6592#ifndef IEM_WITH_CODE_TLB
6593 /* Flush the prefetch buffer. */
6594 pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
6595#endif
6596
6597 return VINF_SUCCESS;
6598}
6599
6600
6601/**
6602 * Get the address of the top of the stack.
6603 *
6604 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6605 */
6606DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
6607{
6608 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6609 return pVCpu->cpum.GstCtx.rsp;
6610 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6611 return pVCpu->cpum.GstCtx.esp;
6612 return pVCpu->cpum.GstCtx.sp;
6613}
6614
6615
6616/**
6617 * Updates the RIP/EIP/IP to point to the next instruction.
6618 *
6619 * This function leaves the EFLAGS.RF flag alone.
6620 *
6621 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6622 * @param cbInstr The number of bytes to add.
6623 */
6624IEM_STATIC void iemRegAddToRipKeepRF(PVMCPU pVCpu, uint8_t cbInstr)
6625{
6626 switch (pVCpu->iem.s.enmCpuMode)
6627 {
6628 case IEMMODE_16BIT:
6629 Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
6630 pVCpu->cpum.GstCtx.eip += cbInstr;
6631 pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
6632 break;
6633
6634 case IEMMODE_32BIT:
6635 pVCpu->cpum.GstCtx.eip += cbInstr;
6636 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
6637 break;
6638
6639 case IEMMODE_64BIT:
6640 pVCpu->cpum.GstCtx.rip += cbInstr;
6641 break;
6642 default: AssertFailed();
6643 }
6644}
6645
6646
6647#if 0
6648/**
6649 * Updates the RIP/EIP/IP to point to the next instruction.
6650 *
6651 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6652 */
6653IEM_STATIC void iemRegUpdateRipKeepRF(PVMCPU pVCpu)
6654{
6655 return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
6656}
6657#endif
6658
6659
6660
6661/**
6662 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
6663 *
6664 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6665 * @param cbInstr The number of bytes to add.
6666 */
6667IEM_STATIC void iemRegAddToRipAndClearRF(PVMCPU pVCpu, uint8_t cbInstr)
6668{
6669 pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
6670
6671 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
6672#if ARCH_BITS >= 64
6673 static uint64_t const s_aRipMasks[] = { UINT64_C(0xffffffff), UINT64_C(0xffffffff), UINT64_MAX };
6674 Assert(pVCpu->cpum.GstCtx.rip <= s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode]);
6675 pVCpu->cpum.GstCtx.rip = (pVCpu->cpum.GstCtx.rip + cbInstr) & s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode];
6676#else
6677 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6678 pVCpu->cpum.GstCtx.rip += cbInstr;
6679 else
6680 pVCpu->cpum.GstCtx.eip += cbInstr;
6681#endif
6682}
6683
6684
6685/**
6686 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
6687 *
6688 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6689 */
6690IEM_STATIC void iemRegUpdateRipAndClearRF(PVMCPU pVCpu)
6691{
6692 return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
6693}
6694
6695
6696/**
6697 * Adds to the stack pointer.
6698 *
6699 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6700 * @param cbToAdd The number of bytes to add (8-bit!).
6701 */
6702DECLINLINE(void) iemRegAddToRsp(PVMCPU pVCpu, uint8_t cbToAdd)
6703{
6704 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6705 pVCpu->cpum.GstCtx.rsp += cbToAdd;
6706 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6707 pVCpu->cpum.GstCtx.esp += cbToAdd;
6708 else
6709 pVCpu->cpum.GstCtx.sp += cbToAdd;
6710}
6711
6712
6713/**
6714 * Subtracts from the stack pointer.
6715 *
6716 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6717 * @param cbToSub The number of bytes to subtract (8-bit!).
6718 */
6719DECLINLINE(void) iemRegSubFromRsp(PVMCPU pVCpu, uint8_t cbToSub)
6720{
6721 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6722 pVCpu->cpum.GstCtx.rsp -= cbToSub;
6723 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6724 pVCpu->cpum.GstCtx.esp -= cbToSub;
6725 else
6726 pVCpu->cpum.GstCtx.sp -= cbToSub;
6727}
6728
6729
6730/**
6731 * Adds to the temporary stack pointer.
6732 *
6733 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6734 * @param pTmpRsp The temporary SP/ESP/RSP to update.
6735 * @param cbToAdd The number of bytes to add (16-bit).
6736 */
6737DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
6738{
6739 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6740 pTmpRsp->u += cbToAdd;
6741 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6742 pTmpRsp->DWords.dw0 += cbToAdd;
6743 else
6744 pTmpRsp->Words.w0 += cbToAdd;
6745}
6746
6747
6748/**
6749 * Subtracts from the temporary stack pointer.
6750 *
6751 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6752 * @param pTmpRsp The temporary SP/ESP/RSP to update.
6753 * @param cbToSub The number of bytes to subtract.
6754 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
6755 * expecting that.
6756 */
6757DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
6758{
6759 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6760 pTmpRsp->u -= cbToSub;
6761 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6762 pTmpRsp->DWords.dw0 -= cbToSub;
6763 else
6764 pTmpRsp->Words.w0 -= cbToSub;
6765}
6766
6767
6768/**
6769 * Calculates the effective stack address for a push of the specified size as
6770 * well as the new RSP value (upper bits may be masked).
6771 *
6772 * @returns Effective stack addressf for the push.
6773 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6774 * @param cbItem The size of the stack item to pop.
6775 * @param puNewRsp Where to return the new RSP value.
6776 */
6777DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
6778{
6779 RTUINT64U uTmpRsp;
6780 RTGCPTR GCPtrTop;
6781 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
6782
6783 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6784 GCPtrTop = uTmpRsp.u -= cbItem;
6785 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6786 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
6787 else
6788 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
6789 *puNewRsp = uTmpRsp.u;
6790 return GCPtrTop;
6791}
6792
6793
6794/**
6795 * Gets the current stack pointer and calculates the value after a pop of the
6796 * specified size.
6797 *
6798 * @returns Current stack pointer.
6799 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6800 * @param cbItem The size of the stack item to pop.
6801 * @param puNewRsp Where to return the new RSP value.
6802 */
6803DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
6804{
6805 RTUINT64U uTmpRsp;
6806 RTGCPTR GCPtrTop;
6807 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
6808
6809 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6810 {
6811 GCPtrTop = uTmpRsp.u;
6812 uTmpRsp.u += cbItem;
6813 }
6814 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6815 {
6816 GCPtrTop = uTmpRsp.DWords.dw0;
6817 uTmpRsp.DWords.dw0 += cbItem;
6818 }
6819 else
6820 {
6821 GCPtrTop = uTmpRsp.Words.w0;
6822 uTmpRsp.Words.w0 += cbItem;
6823 }
6824 *puNewRsp = uTmpRsp.u;
6825 return GCPtrTop;
6826}
6827
6828
6829/**
6830 * Calculates the effective stack address for a push of the specified size as
6831 * well as the new temporary RSP value (upper bits may be masked).
6832 *
6833 * @returns Effective stack addressf for the push.
6834 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6835 * @param pTmpRsp The temporary stack pointer. This is updated.
6836 * @param cbItem The size of the stack item to pop.
6837 */
6838DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
6839{
6840 RTGCPTR GCPtrTop;
6841
6842 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6843 GCPtrTop = pTmpRsp->u -= cbItem;
6844 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6845 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
6846 else
6847 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
6848 return GCPtrTop;
6849}
6850
6851
6852/**
6853 * Gets the effective stack address for a pop of the specified size and
6854 * calculates and updates the temporary RSP.
6855 *
6856 * @returns Current stack pointer.
6857 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6858 * @param pTmpRsp The temporary stack pointer. This is updated.
6859 * @param cbItem The size of the stack item to pop.
6860 */
6861DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
6862{
6863 RTGCPTR GCPtrTop;
6864 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
6865 {
6866 GCPtrTop = pTmpRsp->u;
6867 pTmpRsp->u += cbItem;
6868 }
6869 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
6870 {
6871 GCPtrTop = pTmpRsp->DWords.dw0;
6872 pTmpRsp->DWords.dw0 += cbItem;
6873 }
6874 else
6875 {
6876 GCPtrTop = pTmpRsp->Words.w0;
6877 pTmpRsp->Words.w0 += cbItem;
6878 }
6879 return GCPtrTop;
6880}
6881
6882/** @} */
6883
6884
6885/** @name FPU access and helpers.
6886 *
6887 * @{
6888 */
6889
6890
6891/**
6892 * Hook for preparing to use the host FPU.
6893 *
6894 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6895 *
6896 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6897 */
6898DECLINLINE(void) iemFpuPrepareUsage(PVMCPU pVCpu)
6899{
6900#ifdef IN_RING3
6901 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6902#else
6903 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
6904#endif
6905 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6906}
6907
6908
6909/**
6910 * Hook for preparing to use the host FPU for SSE.
6911 *
6912 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6913 *
6914 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6915 */
6916DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPU pVCpu)
6917{
6918 iemFpuPrepareUsage(pVCpu);
6919}
6920
6921
6922/**
6923 * Hook for preparing to use the host FPU for AVX.
6924 *
6925 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6926 *
6927 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6928 */
6929DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPU pVCpu)
6930{
6931 iemFpuPrepareUsage(pVCpu);
6932}
6933
6934
6935/**
6936 * Hook for actualizing the guest FPU state before the interpreter reads it.
6937 *
6938 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6939 *
6940 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6941 */
6942DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPU pVCpu)
6943{
6944#ifdef IN_RING3
6945 NOREF(pVCpu);
6946#else
6947 CPUMRZFpuStateActualizeForRead(pVCpu);
6948#endif
6949 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6950}
6951
6952
6953/**
6954 * Hook for actualizing the guest FPU state before the interpreter changes it.
6955 *
6956 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6957 *
6958 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6959 */
6960DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPU pVCpu)
6961{
6962#ifdef IN_RING3
6963 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
6964#else
6965 CPUMRZFpuStateActualizeForChange(pVCpu);
6966#endif
6967 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6968}
6969
6970
6971/**
6972 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
6973 * only.
6974 *
6975 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6976 *
6977 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6978 */
6979DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPU pVCpu)
6980{
6981#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
6982 NOREF(pVCpu);
6983#else
6984 CPUMRZFpuStateActualizeSseForRead(pVCpu);
6985#endif
6986 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
6987}
6988
6989
6990/**
6991 * Hook for actualizing the guest XMM0..15 and MXCSR register state for
6992 * read+write.
6993 *
6994 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
6995 *
6996 * @param pVCpu The cross context virtual CPU structure of the calling thread.
6997 */
6998DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPU pVCpu)
6999{
7000#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
7001 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
7002#else
7003 CPUMRZFpuStateActualizeForChange(pVCpu);
7004#endif
7005 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7006}
7007
7008
7009/**
7010 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
7011 * only.
7012 *
7013 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7014 *
7015 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7016 */
7017DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPU pVCpu)
7018{
7019#ifdef IN_RING3
7020 NOREF(pVCpu);
7021#else
7022 CPUMRZFpuStateActualizeAvxForRead(pVCpu);
7023#endif
7024 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7025}
7026
7027
7028/**
7029 * Hook for actualizing the guest YMM0..15 and MXCSR register state for
7030 * read+write.
7031 *
7032 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
7033 *
7034 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7035 */
7036DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPU pVCpu)
7037{
7038#ifdef IN_RING3
7039 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
7040#else
7041 CPUMRZFpuStateActualizeForChange(pVCpu);
7042#endif
7043 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
7044}
7045
7046
7047/**
7048 * Stores a QNaN value into a FPU register.
7049 *
7050 * @param pReg Pointer to the register.
7051 */
7052DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
7053{
7054 pReg->au32[0] = UINT32_C(0x00000000);
7055 pReg->au32[1] = UINT32_C(0xc0000000);
7056 pReg->au16[4] = UINT16_C(0xffff);
7057}
7058
7059
7060/**
7061 * Updates the FOP, FPU.CS and FPUIP registers.
7062 *
7063 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7064 * @param pFpuCtx The FPU context.
7065 */
7066DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPU pVCpu, PX86FXSTATE pFpuCtx)
7067{
7068 Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
7069 pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
7070 /** @todo x87.CS and FPUIP needs to be kept seperately. */
7071 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
7072 {
7073 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
7074 * happens in real mode here based on the fnsave and fnstenv images. */
7075 pFpuCtx->CS = 0;
7076 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
7077 }
7078 else
7079 {
7080 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
7081 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
7082 }
7083}
7084
7085
7086/**
7087 * Updates the x87.DS and FPUDP registers.
7088 *
7089 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7090 * @param pFpuCtx The FPU context.
7091 * @param iEffSeg The effective segment register.
7092 * @param GCPtrEff The effective address relative to @a iEffSeg.
7093 */
7094DECLINLINE(void) iemFpuUpdateDP(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7095{
7096 RTSEL sel;
7097 switch (iEffSeg)
7098 {
7099 case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
7100 case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
7101 case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
7102 case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
7103 case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
7104 case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
7105 default:
7106 AssertMsgFailed(("%d\n", iEffSeg));
7107 sel = pVCpu->cpum.GstCtx.ds.Sel;
7108 }
7109 /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
7110 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
7111 {
7112 pFpuCtx->DS = 0;
7113 pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
7114 }
7115 else
7116 {
7117 pFpuCtx->DS = sel;
7118 pFpuCtx->FPUDP = GCPtrEff;
7119 }
7120}
7121
7122
7123/**
7124 * Rotates the stack registers in the push direction.
7125 *
7126 * @param pFpuCtx The FPU context.
7127 * @remarks This is a complete waste of time, but fxsave stores the registers in
7128 * stack order.
7129 */
7130DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
7131{
7132 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
7133 pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
7134 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
7135 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
7136 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
7137 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
7138 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
7139 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
7140 pFpuCtx->aRegs[0].r80 = r80Tmp;
7141}
7142
7143
7144/**
7145 * Rotates the stack registers in the pop direction.
7146 *
7147 * @param pFpuCtx The FPU context.
7148 * @remarks This is a complete waste of time, but fxsave stores the registers in
7149 * stack order.
7150 */
7151DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
7152{
7153 RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
7154 pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
7155 pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
7156 pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
7157 pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
7158 pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
7159 pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
7160 pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
7161 pFpuCtx->aRegs[7].r80 = r80Tmp;
7162}
7163
7164
7165/**
7166 * Updates FSW and pushes a FPU result onto the FPU stack if no pending
7167 * exception prevents it.
7168 *
7169 * @param pResult The FPU operation result to push.
7170 * @param pFpuCtx The FPU context.
7171 */
7172IEM_STATIC void iemFpuMaybePushResult(PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx)
7173{
7174 /* Update FSW and bail if there are pending exceptions afterwards. */
7175 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
7176 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
7177 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7178 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7179 {
7180 pFpuCtx->FSW = fFsw;
7181 return;
7182 }
7183
7184 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
7185 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
7186 {
7187 /* All is fine, push the actual value. */
7188 pFpuCtx->FTW |= RT_BIT(iNewTop);
7189 pFpuCtx->aRegs[7].r80 = pResult->r80Result;
7190 }
7191 else if (pFpuCtx->FCW & X86_FCW_IM)
7192 {
7193 /* Masked stack overflow, push QNaN. */
7194 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
7195 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7196 }
7197 else
7198 {
7199 /* Raise stack overflow, don't push anything. */
7200 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
7201 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
7202 return;
7203 }
7204
7205 fFsw &= ~X86_FSW_TOP_MASK;
7206 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
7207 pFpuCtx->FSW = fFsw;
7208
7209 iemFpuRotateStackPush(pFpuCtx);
7210}
7211
7212
7213/**
7214 * Stores a result in a FPU register and updates the FSW and FTW.
7215 *
7216 * @param pFpuCtx The FPU context.
7217 * @param pResult The result to store.
7218 * @param iStReg Which FPU register to store it in.
7219 */
7220IEM_STATIC void iemFpuStoreResultOnly(PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg)
7221{
7222 Assert(iStReg < 8);
7223 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7224 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7225 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_TOP_MASK;
7226 pFpuCtx->FTW |= RT_BIT(iReg);
7227 pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
7228}
7229
7230
7231/**
7232 * Only updates the FPU status word (FSW) with the result of the current
7233 * instruction.
7234 *
7235 * @param pFpuCtx The FPU context.
7236 * @param u16FSW The FSW output of the current instruction.
7237 */
7238IEM_STATIC void iemFpuUpdateFSWOnly(PX86FXSTATE pFpuCtx, uint16_t u16FSW)
7239{
7240 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7241 pFpuCtx->FSW |= u16FSW & ~X86_FSW_TOP_MASK;
7242}
7243
7244
7245/**
7246 * Pops one item off the FPU stack if no pending exception prevents it.
7247 *
7248 * @param pFpuCtx The FPU context.
7249 */
7250IEM_STATIC void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx)
7251{
7252 /* Check pending exceptions. */
7253 uint16_t uFSW = pFpuCtx->FSW;
7254 if ( (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7255 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7256 return;
7257
7258 /* TOP--. */
7259 uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
7260 uFSW &= ~X86_FSW_TOP_MASK;
7261 uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7262 pFpuCtx->FSW = uFSW;
7263
7264 /* Mark the previous ST0 as empty. */
7265 iOldTop >>= X86_FSW_TOP_SHIFT;
7266 pFpuCtx->FTW &= ~RT_BIT(iOldTop);
7267
7268 /* Rotate the registers. */
7269 iemFpuRotateStackPop(pFpuCtx);
7270}
7271
7272
7273/**
7274 * Pushes a FPU result onto the FPU stack if no pending exception prevents it.
7275 *
7276 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7277 * @param pResult The FPU operation result to push.
7278 */
7279IEM_STATIC void iemFpuPushResult(PVMCPU pVCpu, PIEMFPURESULT pResult)
7280{
7281 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7282 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7283 iemFpuMaybePushResult(pResult, pFpuCtx);
7284}
7285
7286
7287/**
7288 * Pushes a FPU result onto the FPU stack if no pending exception prevents it,
7289 * and sets FPUDP and FPUDS.
7290 *
7291 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7292 * @param pResult The FPU operation result to push.
7293 * @param iEffSeg The effective segment register.
7294 * @param GCPtrEff The effective address relative to @a iEffSeg.
7295 */
7296IEM_STATIC void iemFpuPushResultWithMemOp(PVMCPU pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7297{
7298 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7299 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7300 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7301 iemFpuMaybePushResult(pResult, pFpuCtx);
7302}
7303
7304
7305/**
7306 * Replace ST0 with the first value and push the second onto the FPU stack,
7307 * unless a pending exception prevents it.
7308 *
7309 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7310 * @param pResult The FPU operation result to store and push.
7311 */
7312IEM_STATIC void iemFpuPushResultTwo(PVMCPU pVCpu, PIEMFPURESULTTWO pResult)
7313{
7314 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7315 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7316
7317 /* Update FSW and bail if there are pending exceptions afterwards. */
7318 uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
7319 fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
7320 if ( (fFsw & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
7321 & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
7322 {
7323 pFpuCtx->FSW = fFsw;
7324 return;
7325 }
7326
7327 uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
7328 if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
7329 {
7330 /* All is fine, push the actual value. */
7331 pFpuCtx->FTW |= RT_BIT(iNewTop);
7332 pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
7333 pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
7334 }
7335 else if (pFpuCtx->FCW & X86_FCW_IM)
7336 {
7337 /* Masked stack overflow, push QNaN. */
7338 fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
7339 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
7340 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7341 }
7342 else
7343 {
7344 /* Raise stack overflow, don't push anything. */
7345 pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
7346 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
7347 return;
7348 }
7349
7350 fFsw &= ~X86_FSW_TOP_MASK;
7351 fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
7352 pFpuCtx->FSW = fFsw;
7353
7354 iemFpuRotateStackPush(pFpuCtx);
7355}
7356
7357
7358/**
7359 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
7360 * FOP.
7361 *
7362 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7363 * @param pResult The result to store.
7364 * @param iStReg Which FPU register to store it in.
7365 */
7366IEM_STATIC void iemFpuStoreResult(PVMCPU pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
7367{
7368 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7369 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7370 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7371}
7372
7373
7374/**
7375 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
7376 * FOP, and then pops the stack.
7377 *
7378 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7379 * @param pResult The result to store.
7380 * @param iStReg Which FPU register to store it in.
7381 */
7382IEM_STATIC void iemFpuStoreResultThenPop(PVMCPU pVCpu, PIEMFPURESULT pResult, uint8_t iStReg)
7383{
7384 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7385 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7386 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7387 iemFpuMaybePopOne(pFpuCtx);
7388}
7389
7390
7391/**
7392 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
7393 * FPUDP, and FPUDS.
7394 *
7395 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7396 * @param pResult The result to store.
7397 * @param iStReg Which FPU register to store it in.
7398 * @param iEffSeg The effective memory operand selector register.
7399 * @param GCPtrEff The effective memory operand offset.
7400 */
7401IEM_STATIC void iemFpuStoreResultWithMemOp(PVMCPU pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
7402 uint8_t iEffSeg, RTGCPTR GCPtrEff)
7403{
7404 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7405 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7406 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7407 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7408}
7409
7410
7411/**
7412 * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
7413 * FPUDP, and FPUDS, and then pops the stack.
7414 *
7415 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7416 * @param pResult The result to store.
7417 * @param iStReg Which FPU register to store it in.
7418 * @param iEffSeg The effective memory operand selector register.
7419 * @param GCPtrEff The effective memory operand offset.
7420 */
7421IEM_STATIC void iemFpuStoreResultWithMemOpThenPop(PVMCPU pVCpu, PIEMFPURESULT pResult,
7422 uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7423{
7424 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7425 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7426 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7427 iemFpuStoreResultOnly(pFpuCtx, pResult, iStReg);
7428 iemFpuMaybePopOne(pFpuCtx);
7429}
7430
7431
7432/**
7433 * Updates the FOP, FPUIP, and FPUCS. For FNOP.
7434 *
7435 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7436 */
7437IEM_STATIC void iemFpuUpdateOpcodeAndIp(PVMCPU pVCpu)
7438{
7439 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7440 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7441}
7442
7443
7444/**
7445 * Marks the specified stack register as free (for FFREE).
7446 *
7447 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7448 * @param iStReg The register to free.
7449 */
7450IEM_STATIC void iemFpuStackFree(PVMCPU pVCpu, uint8_t iStReg)
7451{
7452 Assert(iStReg < 8);
7453 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7454 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7455 pFpuCtx->FTW &= ~RT_BIT(iReg);
7456}
7457
7458
7459/**
7460 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
7461 *
7462 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7463 */
7464IEM_STATIC void iemFpuStackIncTop(PVMCPU pVCpu)
7465{
7466 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7467 uint16_t uFsw = pFpuCtx->FSW;
7468 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
7469 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7470 uFsw &= ~X86_FSW_TOP_MASK;
7471 uFsw |= uTop;
7472 pFpuCtx->FSW = uFsw;
7473}
7474
7475
7476/**
7477 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
7478 *
7479 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7480 */
7481IEM_STATIC void iemFpuStackDecTop(PVMCPU pVCpu)
7482{
7483 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7484 uint16_t uFsw = pFpuCtx->FSW;
7485 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
7486 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
7487 uFsw &= ~X86_FSW_TOP_MASK;
7488 uFsw |= uTop;
7489 pFpuCtx->FSW = uFsw;
7490}
7491
7492
7493/**
7494 * Updates the FSW, FOP, FPUIP, and FPUCS.
7495 *
7496 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7497 * @param u16FSW The FSW from the current instruction.
7498 */
7499IEM_STATIC void iemFpuUpdateFSW(PVMCPU pVCpu, uint16_t u16FSW)
7500{
7501 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7502 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7503 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7504}
7505
7506
7507/**
7508 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
7509 *
7510 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7511 * @param u16FSW The FSW from the current instruction.
7512 */
7513IEM_STATIC void iemFpuUpdateFSWThenPop(PVMCPU pVCpu, uint16_t u16FSW)
7514{
7515 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7516 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7517 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7518 iemFpuMaybePopOne(pFpuCtx);
7519}
7520
7521
7522/**
7523 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
7524 *
7525 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7526 * @param u16FSW The FSW from the current instruction.
7527 * @param iEffSeg The effective memory operand selector register.
7528 * @param GCPtrEff The effective memory operand offset.
7529 */
7530IEM_STATIC void iemFpuUpdateFSWWithMemOp(PVMCPU pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7531{
7532 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7533 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7534 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7535 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7536}
7537
7538
7539/**
7540 * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
7541 *
7542 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7543 * @param u16FSW The FSW from the current instruction.
7544 */
7545IEM_STATIC void iemFpuUpdateFSWThenPopPop(PVMCPU pVCpu, uint16_t u16FSW)
7546{
7547 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7548 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7549 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7550 iemFpuMaybePopOne(pFpuCtx);
7551 iemFpuMaybePopOne(pFpuCtx);
7552}
7553
7554
7555/**
7556 * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
7557 *
7558 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7559 * @param u16FSW The FSW from the current instruction.
7560 * @param iEffSeg The effective memory operand selector register.
7561 * @param GCPtrEff The effective memory operand offset.
7562 */
7563IEM_STATIC void iemFpuUpdateFSWWithMemOpThenPop(PVMCPU pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7564{
7565 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7566 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7567 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7568 iemFpuUpdateFSWOnly(pFpuCtx, u16FSW);
7569 iemFpuMaybePopOne(pFpuCtx);
7570}
7571
7572
7573/**
7574 * Worker routine for raising an FPU stack underflow exception.
7575 *
7576 * @param pFpuCtx The FPU context.
7577 * @param iStReg The stack register being accessed.
7578 */
7579IEM_STATIC void iemFpuStackUnderflowOnly(PX86FXSTATE pFpuCtx, uint8_t iStReg)
7580{
7581 Assert(iStReg < 8 || iStReg == UINT8_MAX);
7582 if (pFpuCtx->FCW & X86_FCW_IM)
7583 {
7584 /* Masked underflow. */
7585 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7586 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7587 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7588 if (iStReg != UINT8_MAX)
7589 {
7590 pFpuCtx->FTW |= RT_BIT(iReg);
7591 iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
7592 }
7593 }
7594 else
7595 {
7596 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7597 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7598 }
7599}
7600
7601
7602/**
7603 * Raises a FPU stack underflow exception.
7604 *
7605 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7606 * @param iStReg The destination register that should be loaded
7607 * with QNaN if \#IS is not masked. Specify
7608 * UINT8_MAX if none (like for fcom).
7609 */
7610DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflow(PVMCPU pVCpu, uint8_t iStReg)
7611{
7612 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7613 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7614 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7615}
7616
7617
7618DECL_NO_INLINE(IEM_STATIC, void)
7619iemFpuStackUnderflowWithMemOp(PVMCPU pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7620{
7621 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7622 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7623 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7624 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7625}
7626
7627
7628DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPop(PVMCPU pVCpu, uint8_t iStReg)
7629{
7630 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7631 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7632 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7633 iemFpuMaybePopOne(pFpuCtx);
7634}
7635
7636
7637DECL_NO_INLINE(IEM_STATIC, void)
7638iemFpuStackUnderflowWithMemOpThenPop(PVMCPU pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7639{
7640 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7641 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7642 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7643 iemFpuStackUnderflowOnly(pFpuCtx, iStReg);
7644 iemFpuMaybePopOne(pFpuCtx);
7645}
7646
7647
7648DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackUnderflowThenPopPop(PVMCPU pVCpu)
7649{
7650 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7651 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7652 iemFpuStackUnderflowOnly(pFpuCtx, UINT8_MAX);
7653 iemFpuMaybePopOne(pFpuCtx);
7654 iemFpuMaybePopOne(pFpuCtx);
7655}
7656
7657
7658DECL_NO_INLINE(IEM_STATIC, void)
7659iemFpuStackPushUnderflow(PVMCPU pVCpu)
7660{
7661 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7662 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7663
7664 if (pFpuCtx->FCW & X86_FCW_IM)
7665 {
7666 /* Masked overflow - Push QNaN. */
7667 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7668 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7669 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7670 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7671 pFpuCtx->FTW |= RT_BIT(iNewTop);
7672 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7673 iemFpuRotateStackPush(pFpuCtx);
7674 }
7675 else
7676 {
7677 /* Exception pending - don't change TOP or the register stack. */
7678 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7679 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7680 }
7681}
7682
7683
7684DECL_NO_INLINE(IEM_STATIC, void)
7685iemFpuStackPushUnderflowTwo(PVMCPU pVCpu)
7686{
7687 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7688 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7689
7690 if (pFpuCtx->FCW & X86_FCW_IM)
7691 {
7692 /* Masked overflow - Push QNaN. */
7693 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7694 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7695 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
7696 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7697 pFpuCtx->FTW |= RT_BIT(iNewTop);
7698 iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
7699 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7700 iemFpuRotateStackPush(pFpuCtx);
7701 }
7702 else
7703 {
7704 /* Exception pending - don't change TOP or the register stack. */
7705 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7706 pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7707 }
7708}
7709
7710
7711/**
7712 * Worker routine for raising an FPU stack overflow exception on a push.
7713 *
7714 * @param pFpuCtx The FPU context.
7715 */
7716IEM_STATIC void iemFpuStackPushOverflowOnly(PX86FXSTATE pFpuCtx)
7717{
7718 if (pFpuCtx->FCW & X86_FCW_IM)
7719 {
7720 /* Masked overflow. */
7721 uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
7722 pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
7723 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
7724 pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
7725 pFpuCtx->FTW |= RT_BIT(iNewTop);
7726 iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
7727 iemFpuRotateStackPush(pFpuCtx);
7728 }
7729 else
7730 {
7731 /* Exception pending - don't change TOP or the register stack. */
7732 pFpuCtx->FSW &= ~X86_FSW_C_MASK;
7733 pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
7734 }
7735}
7736
7737
7738/**
7739 * Raises a FPU stack overflow exception on a push.
7740 *
7741 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7742 */
7743DECL_NO_INLINE(IEM_STATIC, void) iemFpuStackPushOverflow(PVMCPU pVCpu)
7744{
7745 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7746 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7747 iemFpuStackPushOverflowOnly(pFpuCtx);
7748}
7749
7750
7751/**
7752 * Raises a FPU stack overflow exception on a push with a memory operand.
7753 *
7754 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7755 * @param iEffSeg The effective memory operand selector register.
7756 * @param GCPtrEff The effective memory operand offset.
7757 */
7758DECL_NO_INLINE(IEM_STATIC, void)
7759iemFpuStackPushOverflowWithMemOp(PVMCPU pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff)
7760{
7761 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7762 iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
7763 iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
7764 iemFpuStackPushOverflowOnly(pFpuCtx);
7765}
7766
7767
7768IEM_STATIC int iemFpuStRegNotEmpty(PVMCPU pVCpu, uint8_t iStReg)
7769{
7770 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7771 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7772 if (pFpuCtx->FTW & RT_BIT(iReg))
7773 return VINF_SUCCESS;
7774 return VERR_NOT_FOUND;
7775}
7776
7777
7778IEM_STATIC int iemFpuStRegNotEmptyRef(PVMCPU pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
7779{
7780 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7781 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
7782 if (pFpuCtx->FTW & RT_BIT(iReg))
7783 {
7784 *ppRef = &pFpuCtx->aRegs[iStReg].r80;
7785 return VINF_SUCCESS;
7786 }
7787 return VERR_NOT_FOUND;
7788}
7789
7790
7791IEM_STATIC int iemFpu2StRegsNotEmptyRef(PVMCPU pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
7792 uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
7793{
7794 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7795 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7796 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
7797 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
7798 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
7799 {
7800 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
7801 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
7802 return VINF_SUCCESS;
7803 }
7804 return VERR_NOT_FOUND;
7805}
7806
7807
7808IEM_STATIC int iemFpu2StRegsNotEmptyRefFirst(PVMCPU pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
7809{
7810 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
7811 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7812 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
7813 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
7814 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
7815 {
7816 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
7817 return VINF_SUCCESS;
7818 }
7819 return VERR_NOT_FOUND;
7820}
7821
7822
7823/**
7824 * Updates the FPU exception status after FCW is changed.
7825 *
7826 * @param pFpuCtx The FPU context.
7827 */
7828IEM_STATIC void iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
7829{
7830 uint16_t u16Fsw = pFpuCtx->FSW;
7831 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
7832 u16Fsw |= X86_FSW_ES | X86_FSW_B;
7833 else
7834 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
7835 pFpuCtx->FSW = u16Fsw;
7836}
7837
7838
7839/**
7840 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
7841 *
7842 * @returns The full FTW.
7843 * @param pFpuCtx The FPU context.
7844 */
7845IEM_STATIC uint16_t iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
7846{
7847 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
7848 uint16_t u16Ftw = 0;
7849 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
7850 for (unsigned iSt = 0; iSt < 8; iSt++)
7851 {
7852 unsigned const iReg = (iSt + iTop) & 7;
7853 if (!(u8Ftw & RT_BIT(iReg)))
7854 u16Ftw |= 3 << (iReg * 2); /* empty */
7855 else
7856 {
7857 uint16_t uTag;
7858 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
7859 if (pr80Reg->s.uExponent == 0x7fff)
7860 uTag = 2; /* Exponent is all 1's => Special. */
7861 else if (pr80Reg->s.uExponent == 0x0000)
7862 {
7863 if (pr80Reg->s.u64Mantissa == 0x0000)
7864 uTag = 1; /* All bits are zero => Zero. */
7865 else
7866 uTag = 2; /* Must be special. */
7867 }
7868 else if (pr80Reg->s.u64Mantissa & RT_BIT_64(63)) /* The J bit. */
7869 uTag = 0; /* Valid. */
7870 else
7871 uTag = 2; /* Must be special. */
7872
7873 u16Ftw |= uTag << (iReg * 2); /* empty */
7874 }
7875 }
7876
7877 return u16Ftw;
7878}
7879
7880
7881/**
7882 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
7883 *
7884 * @returns The compressed FTW.
7885 * @param u16FullFtw The full FTW to convert.
7886 */
7887IEM_STATIC uint16_t iemFpuCompressFtw(uint16_t u16FullFtw)
7888{
7889 uint8_t u8Ftw = 0;
7890 for (unsigned i = 0; i < 8; i++)
7891 {
7892 if ((u16FullFtw & 3) != 3 /*empty*/)
7893 u8Ftw |= RT_BIT(i);
7894 u16FullFtw >>= 2;
7895 }
7896
7897 return u8Ftw;
7898}
7899
7900/** @} */
7901
7902
7903/** @name Memory access.
7904 *
7905 * @{
7906 */
7907
7908
7909/**
7910 * Updates the IEMCPU::cbWritten counter if applicable.
7911 *
7912 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7913 * @param fAccess The access being accounted for.
7914 * @param cbMem The access size.
7915 */
7916DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPU pVCpu, uint32_t fAccess, size_t cbMem)
7917{
7918 if ( (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE)
7919 || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA | IEM_ACCESS_TYPE_WRITE) )
7920 pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
7921}
7922
7923
7924/**
7925 * Checks if the given segment can be written to, raise the appropriate
7926 * exception if not.
7927 *
7928 * @returns VBox strict status code.
7929 *
7930 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7931 * @param pHid Pointer to the hidden register.
7932 * @param iSegReg The register number.
7933 * @param pu64BaseAddr Where to return the base address to use for the
7934 * segment. (In 64-bit code it may differ from the
7935 * base in the hidden segment.)
7936 */
7937IEM_STATIC VBOXSTRICTRC
7938iemMemSegCheckWriteAccessEx(PVMCPU pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
7939{
7940 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
7941
7942 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7943 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
7944 else
7945 {
7946 if (!pHid->Attr.n.u1Present)
7947 {
7948 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
7949 AssertRelease(uSel == 0);
7950 Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
7951 return iemRaiseGeneralProtectionFault0(pVCpu);
7952 }
7953
7954 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
7955 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
7956 && pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
7957 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
7958 *pu64BaseAddr = pHid->u64Base;
7959 }
7960 return VINF_SUCCESS;
7961}
7962
7963
7964/**
7965 * Checks if the given segment can be read from, raise the appropriate
7966 * exception if not.
7967 *
7968 * @returns VBox strict status code.
7969 *
7970 * @param pVCpu The cross context virtual CPU structure of the calling thread.
7971 * @param pHid Pointer to the hidden register.
7972 * @param iSegReg The register number.
7973 * @param pu64BaseAddr Where to return the base address to use for the
7974 * segment. (In 64-bit code it may differ from the
7975 * base in the hidden segment.)
7976 */
7977IEM_STATIC VBOXSTRICTRC
7978iemMemSegCheckReadAccessEx(PVMCPU pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
7979{
7980 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
7981
7982 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
7983 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
7984 else
7985 {
7986 if (!pHid->Attr.n.u1Present)
7987 {
7988 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
7989 AssertRelease(uSel == 0);
7990 Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
7991 return iemRaiseGeneralProtectionFault0(pVCpu);
7992 }
7993
7994 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
7995 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
7996 *pu64BaseAddr = pHid->u64Base;
7997 }
7998 return VINF_SUCCESS;
7999}
8000
8001
8002/**
8003 * Applies the segment limit, base and attributes.
8004 *
8005 * This may raise a \#GP or \#SS.
8006 *
8007 * @returns VBox strict status code.
8008 *
8009 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8010 * @param fAccess The kind of access which is being performed.
8011 * @param iSegReg The index of the segment register to apply.
8012 * This is UINT8_MAX if none (for IDT, GDT, LDT,
8013 * TSS, ++).
8014 * @param cbMem The access size.
8015 * @param pGCPtrMem Pointer to the guest memory address to apply
8016 * segmentation to. Input and output parameter.
8017 */
8018IEM_STATIC VBOXSTRICTRC
8019iemMemApplySegment(PVMCPU pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem)
8020{
8021 if (iSegReg == UINT8_MAX)
8022 return VINF_SUCCESS;
8023
8024 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
8025 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
8026 switch (pVCpu->iem.s.enmCpuMode)
8027 {
8028 case IEMMODE_16BIT:
8029 case IEMMODE_32BIT:
8030 {
8031 RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
8032 RTGCPTR32 GCPtrLast32 = GCPtrFirst32 + (uint32_t)cbMem - 1;
8033
8034 if ( pSel->Attr.n.u1Present
8035 && !pSel->Attr.n.u1Unusable)
8036 {
8037 Assert(pSel->Attr.n.u1DescType);
8038 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
8039 {
8040 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8041 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
8042 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
8043
8044 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
8045 {
8046 /** @todo CPL check. */
8047 }
8048
8049 /*
8050 * There are two kinds of data selectors, normal and expand down.
8051 */
8052 if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
8053 {
8054 if ( GCPtrFirst32 > pSel->u32Limit
8055 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
8056 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8057 }
8058 else
8059 {
8060 /*
8061 * The upper boundary is defined by the B bit, not the G bit!
8062 */
8063 if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
8064 || GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
8065 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8066 }
8067 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
8068 }
8069 else
8070 {
8071
8072 /*
8073 * Code selector and usually be used to read thru, writing is
8074 * only permitted in real and V8086 mode.
8075 */
8076 if ( ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8077 || ( (fAccess & IEM_ACCESS_TYPE_READ)
8078 && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
8079 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
8080 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
8081
8082 if ( GCPtrFirst32 > pSel->u32Limit
8083 || GCPtrLast32 > pSel->u32Limit) /* yes, in real mode too (since 80286). */
8084 return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
8085
8086 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
8087 {
8088 /** @todo CPL check. */
8089 }
8090
8091 *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
8092 }
8093 }
8094 else
8095 return iemRaiseGeneralProtectionFault0(pVCpu);
8096 return VINF_SUCCESS;
8097 }
8098
8099 case IEMMODE_64BIT:
8100 {
8101 RTGCPTR GCPtrMem = *pGCPtrMem;
8102 if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS)
8103 *pGCPtrMem = GCPtrMem + pSel->u64Base;
8104
8105 Assert(cbMem >= 1);
8106 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
8107 return VINF_SUCCESS;
8108 /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
8109 * 4.12.2 "Data Limit Checks in 64-bit Mode". */
8110 return iemRaiseGeneralProtectionFault0(pVCpu);
8111 }
8112
8113 default:
8114 AssertFailedReturn(VERR_IEM_IPE_7);
8115 }
8116}
8117
8118
8119/**
8120 * Translates a virtual address to a physical physical address and checks if we
8121 * can access the page as specified.
8122 *
8123 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8124 * @param GCPtrMem The virtual address.
8125 * @param fAccess The intended access.
8126 * @param pGCPhysMem Where to return the physical address.
8127 */
8128IEM_STATIC VBOXSTRICTRC
8129iemMemPageTranslateAndCheckAccess(PVMCPU pVCpu, RTGCPTR GCPtrMem, uint32_t fAccess, PRTGCPHYS pGCPhysMem)
8130{
8131 /** @todo Need a different PGM interface here. We're currently using
8132 * generic / REM interfaces. this won't cut it for R0 & RC. */
8133 /** @todo If/when PGM handles paged real-mode, we can remove the hack in
8134 * iemSvmHandleWorldSwitch to work around raising a page-fault here. */
8135 RTGCPHYS GCPhys;
8136 uint64_t fFlags;
8137 int rc = PGMGstGetPage(pVCpu, GCPtrMem, &fFlags, &GCPhys);
8138 if (RT_FAILURE(rc))
8139 {
8140 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
8141 /** @todo Check unassigned memory in unpaged mode. */
8142 /** @todo Reserved bits in page tables. Requires new PGM interface. */
8143 *pGCPhysMem = NIL_RTGCPHYS;
8144 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, rc);
8145 }
8146
8147 /* If the page is writable and does not have the no-exec bit set, all
8148 access is allowed. Otherwise we'll have to check more carefully... */
8149 if ((fFlags & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US))
8150 {
8151 /* Write to read only memory? */
8152 if ( (fAccess & IEM_ACCESS_TYPE_WRITE)
8153 && !(fFlags & X86_PTE_RW)
8154 && ( (pVCpu->iem.s.uCpl == 3
8155 && !(fAccess & IEM_ACCESS_WHAT_SYS))
8156 || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
8157 {
8158 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
8159 *pGCPhysMem = NIL_RTGCPHYS;
8160 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
8161 }
8162
8163 /* Kernel memory accessed by userland? */
8164 if ( !(fFlags & X86_PTE_US)
8165 && pVCpu->iem.s.uCpl == 3
8166 && !(fAccess & IEM_ACCESS_WHAT_SYS))
8167 {
8168 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
8169 *pGCPhysMem = NIL_RTGCPHYS;
8170 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess, VERR_ACCESS_DENIED);
8171 }
8172
8173 /* Executing non-executable memory? */
8174 if ( (fAccess & IEM_ACCESS_TYPE_EXEC)
8175 && (fFlags & X86_PTE_PAE_NX)
8176 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
8177 {
8178 Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
8179 *pGCPhysMem = NIL_RTGCPHYS;
8180 return iemRaisePageFault(pVCpu, GCPtrMem, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE),
8181 VERR_ACCESS_DENIED);
8182 }
8183 }
8184
8185 /*
8186 * Set the dirty / access flags.
8187 * ASSUMES this is set when the address is translated rather than on committ...
8188 */
8189 /** @todo testcase: check when A and D bits are actually set by the CPU. */
8190 uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
8191 if ((fFlags & fAccessedDirty) != fAccessedDirty)
8192 {
8193 int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
8194 AssertRC(rc2);
8195 }
8196
8197 GCPhys |= GCPtrMem & PAGE_OFFSET_MASK;
8198 *pGCPhysMem = GCPhys;
8199 return VINF_SUCCESS;
8200}
8201
8202
8203
8204/**
8205 * Maps a physical page.
8206 *
8207 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
8208 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8209 * @param GCPhysMem The physical address.
8210 * @param fAccess The intended access.
8211 * @param ppvMem Where to return the mapping address.
8212 * @param pLock The PGM lock.
8213 */
8214IEM_STATIC int iemMemPageMap(PVMCPU pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
8215{
8216#ifdef IEM_LOG_MEMORY_WRITES
8217 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8218 return VERR_PGM_PHYS_TLB_CATCH_ALL;
8219#endif
8220
8221 /** @todo This API may require some improving later. A private deal with PGM
8222 * regarding locking and unlocking needs to be struct. A couple of TLBs
8223 * living in PGM, but with publicly accessible inlined access methods
8224 * could perhaps be an even better solution. */
8225 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
8226 GCPhysMem,
8227 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
8228 pVCpu->iem.s.fBypassHandlers,
8229 ppvMem,
8230 pLock);
8231 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
8232 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
8233
8234 return rc;
8235}
8236
8237
8238/**
8239 * Unmap a page previously mapped by iemMemPageMap.
8240 *
8241 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8242 * @param GCPhysMem The physical address.
8243 * @param fAccess The intended access.
8244 * @param pvMem What iemMemPageMap returned.
8245 * @param pLock The PGM lock.
8246 */
8247DECLINLINE(void) iemMemPageUnmap(PVMCPU pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
8248{
8249 NOREF(pVCpu);
8250 NOREF(GCPhysMem);
8251 NOREF(fAccess);
8252 NOREF(pvMem);
8253 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
8254}
8255
8256
8257/**
8258 * Looks up a memory mapping entry.
8259 *
8260 * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
8261 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8262 * @param pvMem The memory address.
8263 * @param fAccess The access to.
8264 */
8265DECLINLINE(int) iemMapLookup(PVMCPU pVCpu, void *pvMem, uint32_t fAccess)
8266{
8267 Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8268 fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;
8269 if ( pVCpu->iem.s.aMemMappings[0].pv == pvMem
8270 && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8271 return 0;
8272 if ( pVCpu->iem.s.aMemMappings[1].pv == pvMem
8273 && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8274 return 1;
8275 if ( pVCpu->iem.s.aMemMappings[2].pv == pvMem
8276 && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
8277 return 2;
8278 return VERR_NOT_FOUND;
8279}
8280
8281
8282/**
8283 * Finds a free memmap entry when using iNextMapping doesn't work.
8284 *
8285 * @returns Memory mapping index, 1024 on failure.
8286 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8287 */
8288IEM_STATIC unsigned iemMemMapFindFree(PVMCPU pVCpu)
8289{
8290 /*
8291 * The easy case.
8292 */
8293 if (pVCpu->iem.s.cActiveMappings == 0)
8294 {
8295 pVCpu->iem.s.iNextMapping = 1;
8296 return 0;
8297 }
8298
8299 /* There should be enough mappings for all instructions. */
8300 AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
8301
8302 for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
8303 if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
8304 return i;
8305
8306 AssertFailedReturn(1024);
8307}
8308
8309
8310/**
8311 * Commits a bounce buffer that needs writing back and unmaps it.
8312 *
8313 * @returns Strict VBox status code.
8314 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8315 * @param iMemMap The index of the buffer to commit.
8316 * @param fPostponeFail Whether we can postpone writer failures to ring-3.
8317 * Always false in ring-3, obviously.
8318 */
8319IEM_STATIC VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPU pVCpu, unsigned iMemMap, bool fPostponeFail)
8320{
8321 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
8322 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
8323#ifdef IN_RING3
8324 Assert(!fPostponeFail);
8325 RT_NOREF_PV(fPostponeFail);
8326#endif
8327
8328 /*
8329 * Do the writing.
8330 */
8331 PVM pVM = pVCpu->CTX_SUFF(pVM);
8332 if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
8333 {
8334 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
8335 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
8336 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8337 if (!pVCpu->iem.s.fBypassHandlers)
8338 {
8339 /*
8340 * Carefully and efficiently dealing with access handler return
8341 * codes make this a little bloated.
8342 */
8343 VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
8344 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
8345 pbBuf,
8346 cbFirst,
8347 PGMACCESSORIGIN_IEM);
8348 if (rcStrict == VINF_SUCCESS)
8349 {
8350 if (cbSecond)
8351 {
8352 rcStrict = PGMPhysWrite(pVM,
8353 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8354 pbBuf + cbFirst,
8355 cbSecond,
8356 PGMACCESSORIGIN_IEM);
8357 if (rcStrict == VINF_SUCCESS)
8358 { /* nothing */ }
8359 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8360 {
8361 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
8362 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8363 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8364 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8365 }
8366#ifndef IN_RING3
8367 else if (fPostponeFail)
8368 {
8369 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8370 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8371 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8372 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
8373 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8374 return iemSetPassUpStatus(pVCpu, rcStrict);
8375 }
8376#endif
8377 else
8378 {
8379 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8380 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8381 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8382 return rcStrict;
8383 }
8384 }
8385 }
8386 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8387 {
8388 if (!cbSecond)
8389 {
8390 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
8391 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8392 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8393 }
8394 else
8395 {
8396 VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
8397 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8398 pbBuf + cbFirst,
8399 cbSecond,
8400 PGMACCESSORIGIN_IEM);
8401 if (rcStrict2 == VINF_SUCCESS)
8402 {
8403 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
8404 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8405 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8406 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8407 }
8408 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
8409 {
8410 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
8411 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8412 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
8413 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
8414 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8415 }
8416#ifndef IN_RING3
8417 else if (fPostponeFail)
8418 {
8419 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8420 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8421 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8422 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
8423 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8424 return iemSetPassUpStatus(pVCpu, rcStrict);
8425 }
8426#endif
8427 else
8428 {
8429 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8430 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8431 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
8432 return rcStrict2;
8433 }
8434 }
8435 }
8436#ifndef IN_RING3
8437 else if (fPostponeFail)
8438 {
8439 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
8440 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8441 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8442 if (!cbSecond)
8443 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;
8444 else
8445 pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;
8446 VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
8447 return iemSetPassUpStatus(pVCpu, rcStrict);
8448 }
8449#endif
8450 else
8451 {
8452 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
8453 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
8454 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8455 return rcStrict;
8456 }
8457 }
8458 else
8459 {
8460 /*
8461 * No access handlers, much simpler.
8462 */
8463 int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
8464 if (RT_SUCCESS(rc))
8465 {
8466 if (cbSecond)
8467 {
8468 rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
8469 if (RT_SUCCESS(rc))
8470 { /* likely */ }
8471 else
8472 {
8473 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
8474 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
8475 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
8476 return rc;
8477 }
8478 }
8479 }
8480 else
8481 {
8482 Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
8483 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
8484 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
8485 return rc;
8486 }
8487 }
8488 }
8489
8490#if defined(IEM_LOG_MEMORY_WRITES)
8491 Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
8492 RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
8493 if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
8494 Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
8495 RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
8496 &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
8497
8498 size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
8499 g_cbIemWrote = cbWrote;
8500 memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
8501#endif
8502
8503 /*
8504 * Free the mapping entry.
8505 */
8506 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8507 Assert(pVCpu->iem.s.cActiveMappings != 0);
8508 pVCpu->iem.s.cActiveMappings--;
8509 return VINF_SUCCESS;
8510}
8511
8512
8513/**
8514 * iemMemMap worker that deals with a request crossing pages.
8515 */
8516IEM_STATIC VBOXSTRICTRC
8517iemMemBounceBufferMapCrossPage(PVMCPU pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
8518{
8519 /*
8520 * Do the address translations.
8521 */
8522 RTGCPHYS GCPhysFirst;
8523 VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, fAccess, &GCPhysFirst);
8524 if (rcStrict != VINF_SUCCESS)
8525 return rcStrict;
8526
8527 RTGCPHYS GCPhysSecond;
8528 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)PAGE_OFFSET_MASK,
8529 fAccess, &GCPhysSecond);
8530 if (rcStrict != VINF_SUCCESS)
8531 return rcStrict;
8532 GCPhysSecond &= ~(RTGCPHYS)PAGE_OFFSET_MASK;
8533
8534 PVM pVM = pVCpu->CTX_SUFF(pVM);
8535
8536 /*
8537 * Read in the current memory content if it's a read, execute or partial
8538 * write access.
8539 */
8540 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8541 uint32_t const cbFirstPage = PAGE_SIZE - (GCPhysFirst & PAGE_OFFSET_MASK);
8542 uint32_t const cbSecondPage = (uint32_t)(cbMem - cbFirstPage);
8543
8544 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
8545 {
8546 if (!pVCpu->iem.s.fBypassHandlers)
8547 {
8548 /*
8549 * Must carefully deal with access handler status codes here,
8550 * makes the code a bit bloated.
8551 */
8552 rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
8553 if (rcStrict == VINF_SUCCESS)
8554 {
8555 rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
8556 if (rcStrict == VINF_SUCCESS)
8557 { /*likely */ }
8558 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8559 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8560 else
8561 {
8562 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
8563 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
8564 return rcStrict;
8565 }
8566 }
8567 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8568 {
8569 VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
8570 if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
8571 {
8572 PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
8573 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8574 }
8575 else
8576 {
8577 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
8578 GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
8579 return rcStrict2;
8580 }
8581 }
8582 else
8583 {
8584 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8585 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8586 return rcStrict;
8587 }
8588 }
8589 else
8590 {
8591 /*
8592 * No informational status codes here, much more straight forward.
8593 */
8594 int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
8595 if (RT_SUCCESS(rc))
8596 {
8597 Assert(rc == VINF_SUCCESS);
8598 rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
8599 if (RT_SUCCESS(rc))
8600 Assert(rc == VINF_SUCCESS);
8601 else
8602 {
8603 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
8604 return rc;
8605 }
8606 }
8607 else
8608 {
8609 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
8610 return rc;
8611 }
8612 }
8613 }
8614#ifdef VBOX_STRICT
8615 else
8616 memset(pbBuf, 0xcc, cbMem);
8617 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
8618 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
8619#endif
8620
8621 /*
8622 * Commit the bounce buffer entry.
8623 */
8624 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
8625 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = GCPhysSecond;
8626 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbFirstPage;
8627 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = (uint16_t)cbSecondPage;
8628 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = false;
8629 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
8630 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
8631 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8632 pVCpu->iem.s.cActiveMappings++;
8633
8634 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8635 *ppvMem = pbBuf;
8636 return VINF_SUCCESS;
8637}
8638
8639
8640/**
8641 * iemMemMap woker that deals with iemMemPageMap failures.
8642 */
8643IEM_STATIC VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPU pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
8644 RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
8645{
8646 /*
8647 * Filter out conditions we can handle and the ones which shouldn't happen.
8648 */
8649 if ( rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
8650 && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
8651 && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
8652 {
8653 AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
8654 return rcMap;
8655 }
8656 pVCpu->iem.s.cPotentialExits++;
8657
8658 /*
8659 * Read in the current memory content if it's a read, execute or partial
8660 * write access.
8661 */
8662 uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
8663 if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
8664 {
8665 if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
8666 memset(pbBuf, 0xff, cbMem);
8667 else
8668 {
8669 int rc;
8670 if (!pVCpu->iem.s.fBypassHandlers)
8671 {
8672 VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
8673 if (rcStrict == VINF_SUCCESS)
8674 { /* nothing */ }
8675 else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
8676 rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
8677 else
8678 {
8679 Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8680 GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
8681 return rcStrict;
8682 }
8683 }
8684 else
8685 {
8686 rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
8687 if (RT_SUCCESS(rc))
8688 { /* likely */ }
8689 else
8690 {
8691 Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
8692 GCPhysFirst, rc));
8693 return rc;
8694 }
8695 }
8696 }
8697 }
8698#ifdef VBOX_STRICT
8699 else
8700 memset(pbBuf, 0xcc, cbMem);
8701#endif
8702#ifdef VBOX_STRICT
8703 if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
8704 memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
8705#endif
8706
8707 /*
8708 * Commit the bounce buffer entry.
8709 */
8710 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst = GCPhysFirst;
8711 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond = NIL_RTGCPHYS;
8712 pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst = (uint16_t)cbMem;
8713 pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond = 0;
8714 pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
8715 pVCpu->iem.s.aMemMappings[iMemMap].pv = pbBuf;
8716 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
8717 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8718 pVCpu->iem.s.cActiveMappings++;
8719
8720 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8721 *ppvMem = pbBuf;
8722 return VINF_SUCCESS;
8723}
8724
8725
8726
8727/**
8728 * Maps the specified guest memory for the given kind of access.
8729 *
8730 * This may be using bounce buffering of the memory if it's crossing a page
8731 * boundary or if there is an access handler installed for any of it. Because
8732 * of lock prefix guarantees, we're in for some extra clutter when this
8733 * happens.
8734 *
8735 * This may raise a \#GP, \#SS, \#PF or \#AC.
8736 *
8737 * @returns VBox strict status code.
8738 *
8739 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8740 * @param ppvMem Where to return the pointer to the mapped
8741 * memory.
8742 * @param cbMem The number of bytes to map. This is usually 1,
8743 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
8744 * string operations it can be up to a page.
8745 * @param iSegReg The index of the segment register to use for
8746 * this access. The base and limits are checked.
8747 * Use UINT8_MAX to indicate that no segmentation
8748 * is required (for IDT, GDT and LDT accesses).
8749 * @param GCPtrMem The address of the guest memory.
8750 * @param fAccess How the memory is being accessed. The
8751 * IEM_ACCESS_TYPE_XXX bit is used to figure out
8752 * how to map the memory, while the
8753 * IEM_ACCESS_WHAT_XXX bit is used when raising
8754 * exceptions.
8755 */
8756IEM_STATIC VBOXSTRICTRC
8757iemMemMap(PVMCPU pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
8758{
8759 /*
8760 * Check the input and figure out which mapping entry to use.
8761 */
8762 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
8763 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
8764 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8765
8766 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
8767 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
8768 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
8769 {
8770 iMemMap = iemMemMapFindFree(pVCpu);
8771 AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
8772 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
8773 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
8774 pVCpu->iem.s.aMemMappings[2].fAccess),
8775 VERR_IEM_IPE_9);
8776 }
8777
8778 /*
8779 * Map the memory, checking that we can actually access it. If something
8780 * slightly complicated happens, fall back on bounce buffering.
8781 */
8782 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
8783 if (rcStrict != VINF_SUCCESS)
8784 return rcStrict;
8785
8786 if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem > PAGE_SIZE) /* Crossing a page boundary? */
8787 return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
8788
8789 RTGCPHYS GCPhysFirst;
8790 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
8791 if (rcStrict != VINF_SUCCESS)
8792 return rcStrict;
8793
8794 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8795 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8796 if (fAccess & IEM_ACCESS_TYPE_READ)
8797 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8798
8799 void *pvMem;
8800 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8801 if (rcStrict != VINF_SUCCESS)
8802 return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
8803
8804 /*
8805 * Fill in the mapping table entry.
8806 */
8807 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
8808 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
8809 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8810 pVCpu->iem.s.cActiveMappings++;
8811
8812 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8813 *ppvMem = pvMem;
8814 return VINF_SUCCESS;
8815}
8816
8817
8818/**
8819 * Commits the guest memory if bounce buffered and unmaps it.
8820 *
8821 * @returns Strict VBox status code.
8822 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8823 * @param pvMem The mapping.
8824 * @param fAccess The kind of access.
8825 */
8826IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPU pVCpu, void *pvMem, uint32_t fAccess)
8827{
8828 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
8829 AssertReturn(iMemMap >= 0, iMemMap);
8830
8831 /* If it's bounce buffered, we may need to write back the buffer. */
8832 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
8833 {
8834 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
8835 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
8836 }
8837 /* Otherwise unlock it. */
8838 else
8839 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8840
8841 /* Free the entry. */
8842 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8843 Assert(pVCpu->iem.s.cActiveMappings != 0);
8844 pVCpu->iem.s.cActiveMappings--;
8845 return VINF_SUCCESS;
8846}
8847
8848#ifdef IEM_WITH_SETJMP
8849
8850/**
8851 * Maps the specified guest memory for the given kind of access, longjmp on
8852 * error.
8853 *
8854 * This may be using bounce buffering of the memory if it's crossing a page
8855 * boundary or if there is an access handler installed for any of it. Because
8856 * of lock prefix guarantees, we're in for some extra clutter when this
8857 * happens.
8858 *
8859 * This may raise a \#GP, \#SS, \#PF or \#AC.
8860 *
8861 * @returns Pointer to the mapped memory.
8862 *
8863 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8864 * @param cbMem The number of bytes to map. This is usually 1,
8865 * 2, 4, 6, 8, 12, 16, 32 or 512. When used by
8866 * string operations it can be up to a page.
8867 * @param iSegReg The index of the segment register to use for
8868 * this access. The base and limits are checked.
8869 * Use UINT8_MAX to indicate that no segmentation
8870 * is required (for IDT, GDT and LDT accesses).
8871 * @param GCPtrMem The address of the guest memory.
8872 * @param fAccess How the memory is being accessed. The
8873 * IEM_ACCESS_TYPE_XXX bit is used to figure out
8874 * how to map the memory, while the
8875 * IEM_ACCESS_WHAT_XXX bit is used when raising
8876 * exceptions.
8877 */
8878IEM_STATIC void *iemMemMapJmp(PVMCPU pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess)
8879{
8880 /*
8881 * Check the input and figure out which mapping entry to use.
8882 */
8883 Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
8884 Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
8885 Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
8886
8887 unsigned iMemMap = pVCpu->iem.s.iNextMapping;
8888 if ( iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
8889 || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
8890 {
8891 iMemMap = iemMemMapFindFree(pVCpu);
8892 AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
8893 ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
8894 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
8895 pVCpu->iem.s.aMemMappings[2].fAccess),
8896 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_9));
8897 }
8898
8899 /*
8900 * Map the memory, checking that we can actually access it. If something
8901 * slightly complicated happens, fall back on bounce buffering.
8902 */
8903 VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
8904 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
8905 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8906
8907 /* Crossing a page boundary? */
8908 if ((GCPtrMem & PAGE_OFFSET_MASK) + cbMem <= PAGE_SIZE)
8909 { /* No (likely). */ }
8910 else
8911 {
8912 void *pvMem;
8913 rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
8914 if (rcStrict == VINF_SUCCESS)
8915 return pvMem;
8916 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8917 }
8918
8919 RTGCPHYS GCPhysFirst;
8920 rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, fAccess, &GCPhysFirst);
8921 if (rcStrict == VINF_SUCCESS) { /*likely*/ }
8922 else longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8923
8924 if (fAccess & IEM_ACCESS_TYPE_WRITE)
8925 Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8926 if (fAccess & IEM_ACCESS_TYPE_READ)
8927 Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
8928
8929 void *pvMem;
8930 rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8931 if (rcStrict == VINF_SUCCESS)
8932 { /* likely */ }
8933 else
8934 {
8935 rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
8936 if (rcStrict == VINF_SUCCESS)
8937 return pvMem;
8938 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8939 }
8940
8941 /*
8942 * Fill in the mapping table entry.
8943 */
8944 pVCpu->iem.s.aMemMappings[iMemMap].pv = pvMem;
8945 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
8946 pVCpu->iem.s.iNextMapping = iMemMap + 1;
8947 pVCpu->iem.s.cActiveMappings++;
8948
8949 iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
8950 return pvMem;
8951}
8952
8953
8954/**
8955 * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
8956 *
8957 * @param pVCpu The cross context virtual CPU structure of the calling thread.
8958 * @param pvMem The mapping.
8959 * @param fAccess The kind of access.
8960 */
8961IEM_STATIC void iemMemCommitAndUnmapJmp(PVMCPU pVCpu, void *pvMem, uint32_t fAccess)
8962{
8963 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
8964 AssertStmt(iMemMap >= 0, longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), iMemMap));
8965
8966 /* If it's bounce buffered, we may need to write back the buffer. */
8967 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
8968 {
8969 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
8970 {
8971 VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
8972 if (rcStrict == VINF_SUCCESS)
8973 return;
8974 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
8975 }
8976 }
8977 /* Otherwise unlock it. */
8978 else
8979 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
8980
8981 /* Free the entry. */
8982 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
8983 Assert(pVCpu->iem.s.cActiveMappings != 0);
8984 pVCpu->iem.s.cActiveMappings--;
8985}
8986
8987#endif /* IEM_WITH_SETJMP */
8988
8989#ifndef IN_RING3
8990/**
8991 * Commits the guest memory if bounce buffered and unmaps it, if any bounce
8992 * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
8993 *
8994 * Allows the instruction to be completed and retired, while the IEM user will
8995 * return to ring-3 immediately afterwards and do the postponed writes there.
8996 *
8997 * @returns VBox status code (no strict statuses). Caller must check
8998 * VMCPU_FF_IEM before repeating string instructions and similar stuff.
8999 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9000 * @param pvMem The mapping.
9001 * @param fAccess The kind of access.
9002 */
9003IEM_STATIC VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPU pVCpu, void *pvMem, uint32_t fAccess)
9004{
9005 int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
9006 AssertReturn(iMemMap >= 0, iMemMap);
9007
9008 /* If it's bounce buffered, we may need to write back the buffer. */
9009 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
9010 {
9011 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
9012 return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);
9013 }
9014 /* Otherwise unlock it. */
9015 else
9016 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9017
9018 /* Free the entry. */
9019 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9020 Assert(pVCpu->iem.s.cActiveMappings != 0);
9021 pVCpu->iem.s.cActiveMappings--;
9022 return VINF_SUCCESS;
9023}
9024#endif
9025
9026
9027/**
9028 * Rollbacks mappings, releasing page locks and such.
9029 *
9030 * The caller shall only call this after checking cActiveMappings.
9031 *
9032 * @returns Strict VBox status code to pass up.
9033 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9034 */
9035IEM_STATIC void iemMemRollback(PVMCPU pVCpu)
9036{
9037 Assert(pVCpu->iem.s.cActiveMappings > 0);
9038
9039 uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
9040 while (iMemMap-- > 0)
9041 {
9042 uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
9043 if (fAccess != IEM_ACCESS_INVALID)
9044 {
9045 AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
9046 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
9047 if (!(fAccess & IEM_ACCESS_BOUNCE_BUFFERED))
9048 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
9049 AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
9050 ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
9051 iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
9052 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
9053 pVCpu->iem.s.cActiveMappings--;
9054 }
9055 }
9056}
9057
9058
9059/**
9060 * Fetches a data byte.
9061 *
9062 * @returns Strict VBox status code.
9063 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9064 * @param pu8Dst Where to return the byte.
9065 * @param iSegReg The index of the segment register to use for
9066 * this access. The base and limits are checked.
9067 * @param GCPtrMem The address of the guest memory.
9068 */
9069IEM_STATIC VBOXSTRICTRC iemMemFetchDataU8(PVMCPU pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9070{
9071 /* The lazy approach for now... */
9072 uint8_t const *pu8Src;
9073 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9074 if (rc == VINF_SUCCESS)
9075 {
9076 *pu8Dst = *pu8Src;
9077 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
9078 }
9079 return rc;
9080}
9081
9082
9083#ifdef IEM_WITH_SETJMP
9084/**
9085 * Fetches a data byte, longjmp on error.
9086 *
9087 * @returns The byte.
9088 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9089 * @param iSegReg The index of the segment register to use for
9090 * this access. The base and limits are checked.
9091 * @param GCPtrMem The address of the guest memory.
9092 */
9093DECL_NO_INLINE(IEM_STATIC, uint8_t) iemMemFetchDataU8Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9094{
9095 /* The lazy approach for now... */
9096 uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9097 uint8_t const bRet = *pu8Src;
9098 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
9099 return bRet;
9100}
9101#endif /* IEM_WITH_SETJMP */
9102
9103
9104/**
9105 * Fetches a data word.
9106 *
9107 * @returns Strict VBox status code.
9108 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9109 * @param pu16Dst Where to return the word.
9110 * @param iSegReg The index of the segment register to use for
9111 * this access. The base and limits are checked.
9112 * @param GCPtrMem The address of the guest memory.
9113 */
9114IEM_STATIC VBOXSTRICTRC iemMemFetchDataU16(PVMCPU pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9115{
9116 /* The lazy approach for now... */
9117 uint16_t const *pu16Src;
9118 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9119 if (rc == VINF_SUCCESS)
9120 {
9121 *pu16Dst = *pu16Src;
9122 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
9123 }
9124 return rc;
9125}
9126
9127
9128#ifdef IEM_WITH_SETJMP
9129/**
9130 * Fetches a data word, longjmp on error.
9131 *
9132 * @returns The word
9133 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9134 * @param iSegReg The index of the segment register to use for
9135 * this access. The base and limits are checked.
9136 * @param GCPtrMem The address of the guest memory.
9137 */
9138DECL_NO_INLINE(IEM_STATIC, uint16_t) iemMemFetchDataU16Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9139{
9140 /* The lazy approach for now... */
9141 uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9142 uint16_t const u16Ret = *pu16Src;
9143 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
9144 return u16Ret;
9145}
9146#endif
9147
9148
9149/**
9150 * Fetches a data dword.
9151 *
9152 * @returns Strict VBox status code.
9153 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9154 * @param pu32Dst Where to return the dword.
9155 * @param iSegReg The index of the segment register to use for
9156 * this access. The base and limits are checked.
9157 * @param GCPtrMem The address of the guest memory.
9158 */
9159IEM_STATIC VBOXSTRICTRC iemMemFetchDataU32(PVMCPU pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9160{
9161 /* The lazy approach for now... */
9162 uint32_t const *pu32Src;
9163 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9164 if (rc == VINF_SUCCESS)
9165 {
9166 *pu32Dst = *pu32Src;
9167 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9168 }
9169 return rc;
9170}
9171
9172
9173#ifdef IEM_WITH_SETJMP
9174
9175IEM_STATIC RTGCPTR iemMemApplySegmentToReadJmp(PVMCPU pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
9176{
9177 Assert(cbMem >= 1);
9178 Assert(iSegReg < X86_SREG_COUNT);
9179
9180 /*
9181 * 64-bit mode is simpler.
9182 */
9183 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9184 {
9185 if (iSegReg >= X86_SREG_FS)
9186 {
9187 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9188 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9189 GCPtrMem += pSel->u64Base;
9190 }
9191
9192 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
9193 return GCPtrMem;
9194 }
9195 /*
9196 * 16-bit and 32-bit segmentation.
9197 */
9198 else
9199 {
9200 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9201 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9202 if ( (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
9203 == X86DESCATTR_P /* data, expand up */
9204 || (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ))
9205 == (X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ) /* code, read-only */ )
9206 {
9207 /* expand up */
9208 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9209 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
9210 && GCPtrLast32 > (uint32_t)GCPtrMem))
9211 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9212 }
9213 else if ( (pSel->Attr.u & (X86DESCATTR_P | X86DESCATTR_UNUSABLE | X86_SEL_TYPE_CODE | X86_SEL_TYPE_DOWN))
9214 == (X86DESCATTR_P | X86_SEL_TYPE_DOWN) /* data, expand down */ )
9215 {
9216 /* expand down */
9217 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9218 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
9219 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
9220 && GCPtrLast32 > (uint32_t)GCPtrMem))
9221 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9222 }
9223 else
9224 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
9225 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
9226 }
9227 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
9228}
9229
9230
9231IEM_STATIC RTGCPTR iemMemApplySegmentToWriteJmp(PVMCPU pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
9232{
9233 Assert(cbMem >= 1);
9234 Assert(iSegReg < X86_SREG_COUNT);
9235
9236 /*
9237 * 64-bit mode is simpler.
9238 */
9239 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9240 {
9241 if (iSegReg >= X86_SREG_FS)
9242 {
9243 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9244 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9245 GCPtrMem += pSel->u64Base;
9246 }
9247
9248 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
9249 return GCPtrMem;
9250 }
9251 /*
9252 * 16-bit and 32-bit segmentation.
9253 */
9254 else
9255 {
9256 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
9257 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
9258 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
9259 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
9260 if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */
9261 {
9262 /* expand up */
9263 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9264 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
9265 && GCPtrLast32 > (uint32_t)GCPtrMem))
9266 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9267 }
9268 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
9269 {
9270 /* expand down */
9271 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
9272 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
9273 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
9274 && GCPtrLast32 > (uint32_t)GCPtrMem))
9275 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
9276 }
9277 else
9278 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
9279 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
9280 }
9281 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
9282}
9283
9284
9285/**
9286 * Fetches a data dword, longjmp on error, fallback/safe version.
9287 *
9288 * @returns The dword
9289 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9290 * @param iSegReg The index of the segment register to use for
9291 * this access. The base and limits are checked.
9292 * @param GCPtrMem The address of the guest memory.
9293 */
9294IEM_STATIC uint32_t iemMemFetchDataU32SafeJmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9295{
9296 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9297 uint32_t const u32Ret = *pu32Src;
9298 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9299 return u32Ret;
9300}
9301
9302
9303/**
9304 * Fetches a data dword, longjmp on error.
9305 *
9306 * @returns The dword
9307 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9308 * @param iSegReg The index of the segment register to use for
9309 * this access. The base and limits are checked.
9310 * @param GCPtrMem The address of the guest memory.
9311 */
9312DECL_NO_INLINE(IEM_STATIC, uint32_t) iemMemFetchDataU32Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9313{
9314# ifdef IEM_WITH_DATA_TLB
9315 RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
9316 if (RT_LIKELY((GCPtrEff & X86_PAGE_OFFSET_MASK) <= X86_PAGE_SIZE - sizeof(uint32_t)))
9317 {
9318 /// @todo more later.
9319 }
9320
9321 return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
9322# else
9323 /* The lazy approach. */
9324 uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9325 uint32_t const u32Ret = *pu32Src;
9326 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
9327 return u32Ret;
9328# endif
9329}
9330#endif
9331
9332
9333#ifdef SOME_UNUSED_FUNCTION
9334/**
9335 * Fetches a data dword and sign extends it to a qword.
9336 *
9337 * @returns Strict VBox status code.
9338 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9339 * @param pu64Dst Where to return the sign extended value.
9340 * @param iSegReg The index of the segment register to use for
9341 * this access. The base and limits are checked.
9342 * @param GCPtrMem The address of the guest memory.
9343 */
9344IEM_STATIC VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPU pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9345{
9346 /* The lazy approach for now... */
9347 int32_t const *pi32Src;
9348 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9349 if (rc == VINF_SUCCESS)
9350 {
9351 *pu64Dst = *pi32Src;
9352 rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
9353 }
9354#ifdef __GNUC__ /* warning: GCC may be a royal pain */
9355 else
9356 *pu64Dst = 0;
9357#endif
9358 return rc;
9359}
9360#endif
9361
9362
9363/**
9364 * Fetches a data qword.
9365 *
9366 * @returns Strict VBox status code.
9367 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9368 * @param pu64Dst Where to return the qword.
9369 * @param iSegReg The index of the segment register to use for
9370 * this access. The base and limits are checked.
9371 * @param GCPtrMem The address of the guest memory.
9372 */
9373IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64(PVMCPU pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9374{
9375 /* The lazy approach for now... */
9376 uint64_t const *pu64Src;
9377 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9378 if (rc == VINF_SUCCESS)
9379 {
9380 *pu64Dst = *pu64Src;
9381 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9382 }
9383 return rc;
9384}
9385
9386
9387#ifdef IEM_WITH_SETJMP
9388/**
9389 * Fetches a data qword, longjmp on error.
9390 *
9391 * @returns The qword.
9392 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9393 * @param iSegReg The index of the segment register to use for
9394 * this access. The base and limits are checked.
9395 * @param GCPtrMem The address of the guest memory.
9396 */
9397DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9398{
9399 /* The lazy approach for now... */
9400 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9401 uint64_t const u64Ret = *pu64Src;
9402 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9403 return u64Ret;
9404}
9405#endif
9406
9407
9408/**
9409 * Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
9410 *
9411 * @returns Strict VBox status code.
9412 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9413 * @param pu64Dst Where to return the qword.
9414 * @param iSegReg The index of the segment register to use for
9415 * this access. The base and limits are checked.
9416 * @param GCPtrMem The address of the guest memory.
9417 */
9418IEM_STATIC VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPU pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9419{
9420 /* The lazy approach for now... */
9421 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9422 if (RT_UNLIKELY(GCPtrMem & 15))
9423 return iemRaiseGeneralProtectionFault0(pVCpu);
9424
9425 uint64_t const *pu64Src;
9426 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9427 if (rc == VINF_SUCCESS)
9428 {
9429 *pu64Dst = *pu64Src;
9430 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9431 }
9432 return rc;
9433}
9434
9435
9436#ifdef IEM_WITH_SETJMP
9437/**
9438 * Fetches a data qword, longjmp on error.
9439 *
9440 * @returns The qword.
9441 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9442 * @param iSegReg The index of the segment register to use for
9443 * this access. The base and limits are checked.
9444 * @param GCPtrMem The address of the guest memory.
9445 */
9446DECL_NO_INLINE(IEM_STATIC, uint64_t) iemMemFetchDataU64AlignedU128Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem)
9447{
9448 /* The lazy approach for now... */
9449 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9450 if (RT_LIKELY(!(GCPtrMem & 15)))
9451 {
9452 uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9453 uint64_t const u64Ret = *pu64Src;
9454 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
9455 return u64Ret;
9456 }
9457
9458 VBOXSTRICTRC rc = iemRaiseGeneralProtectionFault0(pVCpu);
9459 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rc));
9460}
9461#endif
9462
9463
9464/**
9465 * Fetches a data tword.
9466 *
9467 * @returns Strict VBox status code.
9468 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9469 * @param pr80Dst Where to return the tword.
9470 * @param iSegReg The index of the segment register to use for
9471 * this access. The base and limits are checked.
9472 * @param GCPtrMem The address of the guest memory.
9473 */
9474IEM_STATIC VBOXSTRICTRC iemMemFetchDataR80(PVMCPU pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9475{
9476 /* The lazy approach for now... */
9477 PCRTFLOAT80U pr80Src;
9478 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9479 if (rc == VINF_SUCCESS)
9480 {
9481 *pr80Dst = *pr80Src;
9482 rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
9483 }
9484 return rc;
9485}
9486
9487
9488#ifdef IEM_WITH_SETJMP
9489/**
9490 * Fetches a data tword, longjmp on error.
9491 *
9492 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9493 * @param pr80Dst Where to return the tword.
9494 * @param iSegReg The index of the segment register to use for
9495 * this access. The base and limits are checked.
9496 * @param GCPtrMem The address of the guest memory.
9497 */
9498DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataR80Jmp(PVMCPU pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9499{
9500 /* The lazy approach for now... */
9501 PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9502 *pr80Dst = *pr80Src;
9503 iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
9504}
9505#endif
9506
9507
9508/**
9509 * Fetches a data dqword (double qword), generally SSE related.
9510 *
9511 * @returns Strict VBox status code.
9512 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9513 * @param pu128Dst Where to return the qword.
9514 * @param iSegReg The index of the segment register to use for
9515 * this access. The base and limits are checked.
9516 * @param GCPtrMem The address of the guest memory.
9517 */
9518IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128(PVMCPU pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9519{
9520 /* The lazy approach for now... */
9521 PCRTUINT128U pu128Src;
9522 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9523 if (rc == VINF_SUCCESS)
9524 {
9525 pu128Dst->au64[0] = pu128Src->au64[0];
9526 pu128Dst->au64[1] = pu128Src->au64[1];
9527 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9528 }
9529 return rc;
9530}
9531
9532
9533#ifdef IEM_WITH_SETJMP
9534/**
9535 * Fetches a data dqword (double qword), generally SSE related.
9536 *
9537 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9538 * @param pu128Dst Where to return the qword.
9539 * @param iSegReg The index of the segment register to use for
9540 * this access. The base and limits are checked.
9541 * @param GCPtrMem The address of the guest memory.
9542 */
9543IEM_STATIC void iemMemFetchDataU128Jmp(PVMCPU pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9544{
9545 /* The lazy approach for now... */
9546 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9547 pu128Dst->au64[0] = pu128Src->au64[0];
9548 pu128Dst->au64[1] = pu128Src->au64[1];
9549 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9550}
9551#endif
9552
9553
9554/**
9555 * Fetches a data dqword (double qword) at an aligned address, generally SSE
9556 * related.
9557 *
9558 * Raises \#GP(0) if not aligned.
9559 *
9560 * @returns Strict VBox status code.
9561 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9562 * @param pu128Dst Where to return the qword.
9563 * @param iSegReg The index of the segment register to use for
9564 * this access. The base and limits are checked.
9565 * @param GCPtrMem The address of the guest memory.
9566 */
9567IEM_STATIC VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPU pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9568{
9569 /* The lazy approach for now... */
9570 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9571 if ( (GCPtrMem & 15)
9572 && !(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
9573 return iemRaiseGeneralProtectionFault0(pVCpu);
9574
9575 PCRTUINT128U pu128Src;
9576 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9577 if (rc == VINF_SUCCESS)
9578 {
9579 pu128Dst->au64[0] = pu128Src->au64[0];
9580 pu128Dst->au64[1] = pu128Src->au64[1];
9581 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9582 }
9583 return rc;
9584}
9585
9586
9587#ifdef IEM_WITH_SETJMP
9588/**
9589 * Fetches a data dqword (double qword) at an aligned address, generally SSE
9590 * related, longjmp on error.
9591 *
9592 * Raises \#GP(0) if not aligned.
9593 *
9594 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9595 * @param pu128Dst Where to return the qword.
9596 * @param iSegReg The index of the segment register to use for
9597 * this access. The base and limits are checked.
9598 * @param GCPtrMem The address of the guest memory.
9599 */
9600DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU128AlignedSseJmp(PVMCPU pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9601{
9602 /* The lazy approach for now... */
9603 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on SSE stuff. */
9604 if ( (GCPtrMem & 15) == 0
9605 || (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
9606 {
9607 PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9608 pu128Dst->au64[0] = pu128Src->au64[0];
9609 pu128Dst->au64[1] = pu128Src->au64[1];
9610 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
9611 return;
9612 }
9613
9614 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
9615 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9616}
9617#endif
9618
9619
9620/**
9621 * Fetches a data oword (octo word), generally AVX related.
9622 *
9623 * @returns Strict VBox status code.
9624 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9625 * @param pu256Dst Where to return the qword.
9626 * @param iSegReg The index of the segment register to use for
9627 * this access. The base and limits are checked.
9628 * @param GCPtrMem The address of the guest memory.
9629 */
9630IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256(PVMCPU pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9631{
9632 /* The lazy approach for now... */
9633 PCRTUINT256U pu256Src;
9634 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9635 if (rc == VINF_SUCCESS)
9636 {
9637 pu256Dst->au64[0] = pu256Src->au64[0];
9638 pu256Dst->au64[1] = pu256Src->au64[1];
9639 pu256Dst->au64[2] = pu256Src->au64[2];
9640 pu256Dst->au64[3] = pu256Src->au64[3];
9641 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9642 }
9643 return rc;
9644}
9645
9646
9647#ifdef IEM_WITH_SETJMP
9648/**
9649 * Fetches a data oword (octo word), generally AVX related.
9650 *
9651 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9652 * @param pu256Dst Where to return the qword.
9653 * @param iSegReg The index of the segment register to use for
9654 * this access. The base and limits are checked.
9655 * @param GCPtrMem The address of the guest memory.
9656 */
9657IEM_STATIC void iemMemFetchDataU256Jmp(PVMCPU pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9658{
9659 /* The lazy approach for now... */
9660 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9661 pu256Dst->au64[0] = pu256Src->au64[0];
9662 pu256Dst->au64[1] = pu256Src->au64[1];
9663 pu256Dst->au64[2] = pu256Src->au64[2];
9664 pu256Dst->au64[3] = pu256Src->au64[3];
9665 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9666}
9667#endif
9668
9669
9670/**
9671 * Fetches a data oword (octo word) at an aligned address, generally AVX
9672 * related.
9673 *
9674 * Raises \#GP(0) if not aligned.
9675 *
9676 * @returns Strict VBox status code.
9677 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9678 * @param pu256Dst Where to return the qword.
9679 * @param iSegReg The index of the segment register to use for
9680 * this access. The base and limits are checked.
9681 * @param GCPtrMem The address of the guest memory.
9682 */
9683IEM_STATIC VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPU pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9684{
9685 /* The lazy approach for now... */
9686 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
9687 if (GCPtrMem & 31)
9688 return iemRaiseGeneralProtectionFault0(pVCpu);
9689
9690 PCRTUINT256U pu256Src;
9691 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9692 if (rc == VINF_SUCCESS)
9693 {
9694 pu256Dst->au64[0] = pu256Src->au64[0];
9695 pu256Dst->au64[1] = pu256Src->au64[1];
9696 pu256Dst->au64[2] = pu256Src->au64[2];
9697 pu256Dst->au64[3] = pu256Src->au64[3];
9698 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9699 }
9700 return rc;
9701}
9702
9703
9704#ifdef IEM_WITH_SETJMP
9705/**
9706 * Fetches a data oword (octo word) at an aligned address, generally AVX
9707 * related, longjmp on error.
9708 *
9709 * Raises \#GP(0) if not aligned.
9710 *
9711 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9712 * @param pu256Dst Where to return the qword.
9713 * @param iSegReg The index of the segment register to use for
9714 * this access. The base and limits are checked.
9715 * @param GCPtrMem The address of the guest memory.
9716 */
9717DECL_NO_INLINE(IEM_STATIC, void) iemMemFetchDataU256AlignedSseJmp(PVMCPU pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
9718{
9719 /* The lazy approach for now... */
9720 /** @todo testcase: Ordering of \#SS(0) vs \#GP() vs \#PF on AVX stuff. */
9721 if ((GCPtrMem & 31) == 0)
9722 {
9723 PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R);
9724 pu256Dst->au64[0] = pu256Src->au64[0];
9725 pu256Dst->au64[1] = pu256Src->au64[1];
9726 pu256Dst->au64[2] = pu256Src->au64[2];
9727 pu256Dst->au64[3] = pu256Src->au64[3];
9728 iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
9729 return;
9730 }
9731
9732 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
9733 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
9734}
9735#endif
9736
9737
9738
9739/**
9740 * Fetches a descriptor register (lgdt, lidt).
9741 *
9742 * @returns Strict VBox status code.
9743 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9744 * @param pcbLimit Where to return the limit.
9745 * @param pGCPtrBase Where to return the base.
9746 * @param iSegReg The index of the segment register to use for
9747 * this access. The base and limits are checked.
9748 * @param GCPtrMem The address of the guest memory.
9749 * @param enmOpSize The effective operand size.
9750 */
9751IEM_STATIC VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPU pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
9752 RTGCPTR GCPtrMem, IEMMODE enmOpSize)
9753{
9754 /*
9755 * Just like SIDT and SGDT, the LIDT and LGDT instructions are a
9756 * little special:
9757 * - The two reads are done separately.
9758 * - Operand size override works in 16-bit and 32-bit code, but 64-bit.
9759 * - We suspect the 386 to actually commit the limit before the base in
9760 * some cases (search for 386 in bs3CpuBasic2_lidt_lgdt_One). We
9761 * don't try emulate this eccentric behavior, because it's not well
9762 * enough understood and rather hard to trigger.
9763 * - The 486 seems to do a dword limit read when the operand size is 32-bit.
9764 */
9765 VBOXSTRICTRC rcStrict;
9766 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
9767 {
9768 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9769 if (rcStrict == VINF_SUCCESS)
9770 rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
9771 }
9772 else
9773 {
9774 uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
9775 if (enmOpSize == IEMMODE_32BIT)
9776 {
9777 if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
9778 {
9779 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9780 if (rcStrict == VINF_SUCCESS)
9781 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9782 }
9783 else
9784 {
9785 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
9786 if (rcStrict == VINF_SUCCESS)
9787 {
9788 *pcbLimit = (uint16_t)uTmp;
9789 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9790 }
9791 }
9792 if (rcStrict == VINF_SUCCESS)
9793 *pGCPtrBase = uTmp;
9794 }
9795 else
9796 {
9797 rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
9798 if (rcStrict == VINF_SUCCESS)
9799 {
9800 rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
9801 if (rcStrict == VINF_SUCCESS)
9802 *pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
9803 }
9804 }
9805 }
9806 return rcStrict;
9807}
9808
9809
9810
9811/**
9812 * Stores a data byte.
9813 *
9814 * @returns Strict VBox status code.
9815 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9816 * @param iSegReg The index of the segment register to use for
9817 * this access. The base and limits are checked.
9818 * @param GCPtrMem The address of the guest memory.
9819 * @param u8Value The value to store.
9820 */
9821IEM_STATIC VBOXSTRICTRC iemMemStoreDataU8(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
9822{
9823 /* The lazy approach for now... */
9824 uint8_t *pu8Dst;
9825 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9826 if (rc == VINF_SUCCESS)
9827 {
9828 *pu8Dst = u8Value;
9829 rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
9830 }
9831 return rc;
9832}
9833
9834
9835#ifdef IEM_WITH_SETJMP
9836/**
9837 * Stores a data byte, longjmp on error.
9838 *
9839 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9840 * @param iSegReg The index of the segment register to use for
9841 * this access. The base and limits are checked.
9842 * @param GCPtrMem The address of the guest memory.
9843 * @param u8Value The value to store.
9844 */
9845IEM_STATIC void iemMemStoreDataU8Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value)
9846{
9847 /* The lazy approach for now... */
9848 uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9849 *pu8Dst = u8Value;
9850 iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
9851}
9852#endif
9853
9854
9855/**
9856 * Stores a data word.
9857 *
9858 * @returns Strict VBox status code.
9859 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9860 * @param iSegReg The index of the segment register to use for
9861 * this access. The base and limits are checked.
9862 * @param GCPtrMem The address of the guest memory.
9863 * @param u16Value The value to store.
9864 */
9865IEM_STATIC VBOXSTRICTRC iemMemStoreDataU16(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
9866{
9867 /* The lazy approach for now... */
9868 uint16_t *pu16Dst;
9869 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9870 if (rc == VINF_SUCCESS)
9871 {
9872 *pu16Dst = u16Value;
9873 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
9874 }
9875 return rc;
9876}
9877
9878
9879#ifdef IEM_WITH_SETJMP
9880/**
9881 * Stores a data word, longjmp on error.
9882 *
9883 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9884 * @param iSegReg The index of the segment register to use for
9885 * this access. The base and limits are checked.
9886 * @param GCPtrMem The address of the guest memory.
9887 * @param u16Value The value to store.
9888 */
9889IEM_STATIC void iemMemStoreDataU16Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value)
9890{
9891 /* The lazy approach for now... */
9892 uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9893 *pu16Dst = u16Value;
9894 iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
9895}
9896#endif
9897
9898
9899/**
9900 * Stores a data dword.
9901 *
9902 * @returns Strict VBox status code.
9903 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9904 * @param iSegReg The index of the segment register to use for
9905 * this access. The base and limits are checked.
9906 * @param GCPtrMem The address of the guest memory.
9907 * @param u32Value The value to store.
9908 */
9909IEM_STATIC VBOXSTRICTRC iemMemStoreDataU32(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
9910{
9911 /* The lazy approach for now... */
9912 uint32_t *pu32Dst;
9913 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9914 if (rc == VINF_SUCCESS)
9915 {
9916 *pu32Dst = u32Value;
9917 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
9918 }
9919 return rc;
9920}
9921
9922
9923#ifdef IEM_WITH_SETJMP
9924/**
9925 * Stores a data dword.
9926 *
9927 * @returns Strict VBox status code.
9928 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9929 * @param iSegReg The index of the segment register to use for
9930 * this access. The base and limits are checked.
9931 * @param GCPtrMem The address of the guest memory.
9932 * @param u32Value The value to store.
9933 */
9934IEM_STATIC void iemMemStoreDataU32Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value)
9935{
9936 /* The lazy approach for now... */
9937 uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9938 *pu32Dst = u32Value;
9939 iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
9940}
9941#endif
9942
9943
9944/**
9945 * Stores a data qword.
9946 *
9947 * @returns Strict VBox status code.
9948 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9949 * @param iSegReg The index of the segment register to use for
9950 * this access. The base and limits are checked.
9951 * @param GCPtrMem The address of the guest memory.
9952 * @param u64Value The value to store.
9953 */
9954IEM_STATIC VBOXSTRICTRC iemMemStoreDataU64(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
9955{
9956 /* The lazy approach for now... */
9957 uint64_t *pu64Dst;
9958 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9959 if (rc == VINF_SUCCESS)
9960 {
9961 *pu64Dst = u64Value;
9962 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
9963 }
9964 return rc;
9965}
9966
9967
9968#ifdef IEM_WITH_SETJMP
9969/**
9970 * Stores a data qword, longjmp on error.
9971 *
9972 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9973 * @param iSegReg The index of the segment register to use for
9974 * this access. The base and limits are checked.
9975 * @param GCPtrMem The address of the guest memory.
9976 * @param u64Value The value to store.
9977 */
9978IEM_STATIC void iemMemStoreDataU64Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value)
9979{
9980 /* The lazy approach for now... */
9981 uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
9982 *pu64Dst = u64Value;
9983 iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
9984}
9985#endif
9986
9987
9988/**
9989 * Stores a data dqword.
9990 *
9991 * @returns Strict VBox status code.
9992 * @param pVCpu The cross context virtual CPU structure of the calling thread.
9993 * @param iSegReg The index of the segment register to use for
9994 * this access. The base and limits are checked.
9995 * @param GCPtrMem The address of the guest memory.
9996 * @param u128Value The value to store.
9997 */
9998IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
9999{
10000 /* The lazy approach for now... */
10001 PRTUINT128U pu128Dst;
10002 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10003 if (rc == VINF_SUCCESS)
10004 {
10005 pu128Dst->au64[0] = u128Value.au64[0];
10006 pu128Dst->au64[1] = u128Value.au64[1];
10007 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10008 }
10009 return rc;
10010}
10011
10012
10013#ifdef IEM_WITH_SETJMP
10014/**
10015 * Stores a data dqword, longjmp on error.
10016 *
10017 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10018 * @param iSegReg The index of the segment register to use for
10019 * this access. The base and limits are checked.
10020 * @param GCPtrMem The address of the guest memory.
10021 * @param u128Value The value to store.
10022 */
10023IEM_STATIC void iemMemStoreDataU128Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10024{
10025 /* The lazy approach for now... */
10026 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10027 pu128Dst->au64[0] = u128Value.au64[0];
10028 pu128Dst->au64[1] = u128Value.au64[1];
10029 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10030}
10031#endif
10032
10033
10034/**
10035 * Stores a data dqword, SSE aligned.
10036 *
10037 * @returns Strict VBox status code.
10038 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10039 * @param iSegReg The index of the segment register to use for
10040 * this access. The base and limits are checked.
10041 * @param GCPtrMem The address of the guest memory.
10042 * @param u128Value The value to store.
10043 */
10044IEM_STATIC VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10045{
10046 /* The lazy approach for now... */
10047 if ( (GCPtrMem & 15)
10048 && !(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
10049 return iemRaiseGeneralProtectionFault0(pVCpu);
10050
10051 PRTUINT128U pu128Dst;
10052 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10053 if (rc == VINF_SUCCESS)
10054 {
10055 pu128Dst->au64[0] = u128Value.au64[0];
10056 pu128Dst->au64[1] = u128Value.au64[1];
10057 rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10058 }
10059 return rc;
10060}
10061
10062
10063#ifdef IEM_WITH_SETJMP
10064/**
10065 * Stores a data dqword, SSE aligned.
10066 *
10067 * @returns Strict VBox status code.
10068 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10069 * @param iSegReg The index of the segment register to use for
10070 * this access. The base and limits are checked.
10071 * @param GCPtrMem The address of the guest memory.
10072 * @param u128Value The value to store.
10073 */
10074DECL_NO_INLINE(IEM_STATIC, void)
10075iemMemStoreDataU128AlignedSseJmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value)
10076{
10077 /* The lazy approach for now... */
10078 if ( (GCPtrMem & 15) == 0
10079 || (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.MXCSR & X86_MXCSR_MM)) /** @todo should probably check this *after* applying seg.u64Base... Check real HW. */
10080 {
10081 PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10082 pu128Dst->au64[0] = u128Value.au64[0];
10083 pu128Dst->au64[1] = u128Value.au64[1];
10084 iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
10085 return;
10086 }
10087
10088 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
10089 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
10090}
10091#endif
10092
10093
10094/**
10095 * Stores a data dqword.
10096 *
10097 * @returns Strict VBox status code.
10098 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10099 * @param iSegReg The index of the segment register to use for
10100 * this access. The base and limits are checked.
10101 * @param GCPtrMem The address of the guest memory.
10102 * @param pu256Value Pointer to the value to store.
10103 */
10104IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10105{
10106 /* The lazy approach for now... */
10107 PRTUINT256U pu256Dst;
10108 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10109 if (rc == VINF_SUCCESS)
10110 {
10111 pu256Dst->au64[0] = pu256Value->au64[0];
10112 pu256Dst->au64[1] = pu256Value->au64[1];
10113 pu256Dst->au64[2] = pu256Value->au64[2];
10114 pu256Dst->au64[3] = pu256Value->au64[3];
10115 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10116 }
10117 return rc;
10118}
10119
10120
10121#ifdef IEM_WITH_SETJMP
10122/**
10123 * Stores a data dqword, longjmp on error.
10124 *
10125 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10126 * @param iSegReg The index of the segment register to use for
10127 * this access. The base and limits are checked.
10128 * @param GCPtrMem The address of the guest memory.
10129 * @param pu256Value Pointer to the value to store.
10130 */
10131IEM_STATIC void iemMemStoreDataU256Jmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10132{
10133 /* The lazy approach for now... */
10134 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10135 pu256Dst->au64[0] = pu256Value->au64[0];
10136 pu256Dst->au64[1] = pu256Value->au64[1];
10137 pu256Dst->au64[2] = pu256Value->au64[2];
10138 pu256Dst->au64[3] = pu256Value->au64[3];
10139 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10140}
10141#endif
10142
10143
10144/**
10145 * Stores a data dqword, AVX aligned.
10146 *
10147 * @returns Strict VBox status code.
10148 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10149 * @param iSegReg The index of the segment register to use for
10150 * this access. The base and limits are checked.
10151 * @param GCPtrMem The address of the guest memory.
10152 * @param pu256Value Pointer to the value to store.
10153 */
10154IEM_STATIC VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10155{
10156 /* The lazy approach for now... */
10157 if (GCPtrMem & 31)
10158 return iemRaiseGeneralProtectionFault0(pVCpu);
10159
10160 PRTUINT256U pu256Dst;
10161 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10162 if (rc == VINF_SUCCESS)
10163 {
10164 pu256Dst->au64[0] = pu256Value->au64[0];
10165 pu256Dst->au64[1] = pu256Value->au64[1];
10166 pu256Dst->au64[2] = pu256Value->au64[2];
10167 pu256Dst->au64[3] = pu256Value->au64[3];
10168 rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10169 }
10170 return rc;
10171}
10172
10173
10174#ifdef IEM_WITH_SETJMP
10175/**
10176 * Stores a data dqword, AVX aligned.
10177 *
10178 * @returns Strict VBox status code.
10179 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10180 * @param iSegReg The index of the segment register to use for
10181 * this access. The base and limits are checked.
10182 * @param GCPtrMem The address of the guest memory.
10183 * @param pu256Value Pointer to the value to store.
10184 */
10185DECL_NO_INLINE(IEM_STATIC, void)
10186iemMemStoreDataU256AlignedAvxJmp(PVMCPU pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value)
10187{
10188 /* The lazy approach for now... */
10189 if ((GCPtrMem & 31) == 0)
10190 {
10191 PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W);
10192 pu256Dst->au64[0] = pu256Value->au64[0];
10193 pu256Dst->au64[1] = pu256Value->au64[1];
10194 pu256Dst->au64[2] = pu256Value->au64[2];
10195 pu256Dst->au64[3] = pu256Value->au64[3];
10196 iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
10197 return;
10198 }
10199
10200 VBOXSTRICTRC rcStrict = iemRaiseGeneralProtectionFault0(pVCpu);
10201 longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VBOXSTRICTRC_VAL(rcStrict));
10202}
10203#endif
10204
10205
10206/**
10207 * Stores a descriptor register (sgdt, sidt).
10208 *
10209 * @returns Strict VBox status code.
10210 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10211 * @param cbLimit The limit.
10212 * @param GCPtrBase The base address.
10213 * @param iSegReg The index of the segment register to use for
10214 * this access. The base and limits are checked.
10215 * @param GCPtrMem The address of the guest memory.
10216 */
10217IEM_STATIC VBOXSTRICTRC
10218iemMemStoreDataXdtr(PVMCPU pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem)
10219{
10220 /*
10221 * The SIDT and SGDT instructions actually stores the data using two
10222 * independent writes. The instructions does not respond to opsize prefixes.
10223 */
10224 VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
10225 if (rcStrict == VINF_SUCCESS)
10226 {
10227 if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
10228 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
10229 IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
10230 ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
10231 else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
10232 rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
10233 else
10234 rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
10235 }
10236 return rcStrict;
10237}
10238
10239
10240/**
10241 * Pushes a word onto the stack.
10242 *
10243 * @returns Strict VBox status code.
10244 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10245 * @param u16Value The value to push.
10246 */
10247IEM_STATIC VBOXSTRICTRC iemMemStackPushU16(PVMCPU pVCpu, uint16_t u16Value)
10248{
10249 /* Increment the stack pointer. */
10250 uint64_t uNewRsp;
10251 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
10252
10253 /* Write the word the lazy way. */
10254 uint16_t *pu16Dst;
10255 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10256 if (rc == VINF_SUCCESS)
10257 {
10258 *pu16Dst = u16Value;
10259 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
10260 }
10261
10262 /* Commit the new RSP value unless we an access handler made trouble. */
10263 if (rc == VINF_SUCCESS)
10264 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10265
10266 return rc;
10267}
10268
10269
10270/**
10271 * Pushes a dword onto the stack.
10272 *
10273 * @returns Strict VBox status code.
10274 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10275 * @param u32Value The value to push.
10276 */
10277IEM_STATIC VBOXSTRICTRC iemMemStackPushU32(PVMCPU pVCpu, uint32_t u32Value)
10278{
10279 /* Increment the stack pointer. */
10280 uint64_t uNewRsp;
10281 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
10282
10283 /* Write the dword the lazy way. */
10284 uint32_t *pu32Dst;
10285 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10286 if (rc == VINF_SUCCESS)
10287 {
10288 *pu32Dst = u32Value;
10289 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
10290 }
10291
10292 /* Commit the new RSP value unless we an access handler made trouble. */
10293 if (rc == VINF_SUCCESS)
10294 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10295
10296 return rc;
10297}
10298
10299
10300/**
10301 * Pushes a dword segment register value onto the stack.
10302 *
10303 * @returns Strict VBox status code.
10304 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10305 * @param u32Value The value to push.
10306 */
10307IEM_STATIC VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPU pVCpu, uint32_t u32Value)
10308{
10309 /* Increment the stack pointer. */
10310 uint64_t uNewRsp;
10311 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
10312
10313 /* The intel docs talks about zero extending the selector register
10314 value. My actual intel CPU here might be zero extending the value
10315 but it still only writes the lower word... */
10316 /** @todo Test this on new HW and on AMD and in 64-bit mode. Also test what
10317 * happens when crossing an electric page boundrary, is the high word checked
10318 * for write accessibility or not? Probably it is. What about segment limits?
10319 * It appears this behavior is also shared with trap error codes.
10320 *
10321 * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
10322 * ancient hardware when it actually did change. */
10323 uint16_t *pu16Dst;
10324 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_RW);
10325 if (rc == VINF_SUCCESS)
10326 {
10327 *pu16Dst = (uint16_t)u32Value;
10328 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
10329 }
10330
10331 /* Commit the new RSP value unless we an access handler made trouble. */
10332 if (rc == VINF_SUCCESS)
10333 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10334
10335 return rc;
10336}
10337
10338
10339/**
10340 * Pushes a qword onto the stack.
10341 *
10342 * @returns Strict VBox status code.
10343 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10344 * @param u64Value The value to push.
10345 */
10346IEM_STATIC VBOXSTRICTRC iemMemStackPushU64(PVMCPU pVCpu, uint64_t u64Value)
10347{
10348 /* Increment the stack pointer. */
10349 uint64_t uNewRsp;
10350 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
10351
10352 /* Write the word the lazy way. */
10353 uint64_t *pu64Dst;
10354 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10355 if (rc == VINF_SUCCESS)
10356 {
10357 *pu64Dst = u64Value;
10358 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
10359 }
10360
10361 /* Commit the new RSP value unless we an access handler made trouble. */
10362 if (rc == VINF_SUCCESS)
10363 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10364
10365 return rc;
10366}
10367
10368
10369/**
10370 * Pops a word from the stack.
10371 *
10372 * @returns Strict VBox status code.
10373 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10374 * @param pu16Value Where to store the popped value.
10375 */
10376IEM_STATIC VBOXSTRICTRC iemMemStackPopU16(PVMCPU pVCpu, uint16_t *pu16Value)
10377{
10378 /* Increment the stack pointer. */
10379 uint64_t uNewRsp;
10380 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
10381
10382 /* Write the word the lazy way. */
10383 uint16_t const *pu16Src;
10384 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10385 if (rc == VINF_SUCCESS)
10386 {
10387 *pu16Value = *pu16Src;
10388 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
10389
10390 /* Commit the new RSP value. */
10391 if (rc == VINF_SUCCESS)
10392 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10393 }
10394
10395 return rc;
10396}
10397
10398
10399/**
10400 * Pops a dword from the stack.
10401 *
10402 * @returns Strict VBox status code.
10403 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10404 * @param pu32Value Where to store the popped value.
10405 */
10406IEM_STATIC VBOXSTRICTRC iemMemStackPopU32(PVMCPU pVCpu, uint32_t *pu32Value)
10407{
10408 /* Increment the stack pointer. */
10409 uint64_t uNewRsp;
10410 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
10411
10412 /* Write the word the lazy way. */
10413 uint32_t const *pu32Src;
10414 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10415 if (rc == VINF_SUCCESS)
10416 {
10417 *pu32Value = *pu32Src;
10418 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
10419
10420 /* Commit the new RSP value. */
10421 if (rc == VINF_SUCCESS)
10422 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10423 }
10424
10425 return rc;
10426}
10427
10428
10429/**
10430 * Pops a qword from the stack.
10431 *
10432 * @returns Strict VBox status code.
10433 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10434 * @param pu64Value Where to store the popped value.
10435 */
10436IEM_STATIC VBOXSTRICTRC iemMemStackPopU64(PVMCPU pVCpu, uint64_t *pu64Value)
10437{
10438 /* Increment the stack pointer. */
10439 uint64_t uNewRsp;
10440 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
10441
10442 /* Write the word the lazy way. */
10443 uint64_t const *pu64Src;
10444 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10445 if (rc == VINF_SUCCESS)
10446 {
10447 *pu64Value = *pu64Src;
10448 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
10449
10450 /* Commit the new RSP value. */
10451 if (rc == VINF_SUCCESS)
10452 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10453 }
10454
10455 return rc;
10456}
10457
10458
10459/**
10460 * Pushes a word onto the stack, using a temporary stack pointer.
10461 *
10462 * @returns Strict VBox status code.
10463 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10464 * @param u16Value The value to push.
10465 * @param pTmpRsp Pointer to the temporary stack pointer.
10466 */
10467IEM_STATIC VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPU pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp)
10468{
10469 /* Increment the stack pointer. */
10470 RTUINT64U NewRsp = *pTmpRsp;
10471 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
10472
10473 /* Write the word the lazy way. */
10474 uint16_t *pu16Dst;
10475 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10476 if (rc == VINF_SUCCESS)
10477 {
10478 *pu16Dst = u16Value;
10479 rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
10480 }
10481
10482 /* Commit the new RSP value unless we an access handler made trouble. */
10483 if (rc == VINF_SUCCESS)
10484 *pTmpRsp = NewRsp;
10485
10486 return rc;
10487}
10488
10489
10490/**
10491 * Pushes a dword onto the stack, using a temporary stack pointer.
10492 *
10493 * @returns Strict VBox status code.
10494 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10495 * @param u32Value The value to push.
10496 * @param pTmpRsp Pointer to the temporary stack pointer.
10497 */
10498IEM_STATIC VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPU pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp)
10499{
10500 /* Increment the stack pointer. */
10501 RTUINT64U NewRsp = *pTmpRsp;
10502 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
10503
10504 /* Write the word the lazy way. */
10505 uint32_t *pu32Dst;
10506 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10507 if (rc == VINF_SUCCESS)
10508 {
10509 *pu32Dst = u32Value;
10510 rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
10511 }
10512
10513 /* Commit the new RSP value unless we an access handler made trouble. */
10514 if (rc == VINF_SUCCESS)
10515 *pTmpRsp = NewRsp;
10516
10517 return rc;
10518}
10519
10520
10521/**
10522 * Pushes a dword onto the stack, using a temporary stack pointer.
10523 *
10524 * @returns Strict VBox status code.
10525 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10526 * @param u64Value The value to push.
10527 * @param pTmpRsp Pointer to the temporary stack pointer.
10528 */
10529IEM_STATIC VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPU pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp)
10530{
10531 /* Increment the stack pointer. */
10532 RTUINT64U NewRsp = *pTmpRsp;
10533 RTGCPTR GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
10534
10535 /* Write the word the lazy way. */
10536 uint64_t *pu64Dst;
10537 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10538 if (rc == VINF_SUCCESS)
10539 {
10540 *pu64Dst = u64Value;
10541 rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
10542 }
10543
10544 /* Commit the new RSP value unless we an access handler made trouble. */
10545 if (rc == VINF_SUCCESS)
10546 *pTmpRsp = NewRsp;
10547
10548 return rc;
10549}
10550
10551
10552/**
10553 * Pops a word from the stack, using a temporary stack pointer.
10554 *
10555 * @returns Strict VBox status code.
10556 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10557 * @param pu16Value Where to store the popped value.
10558 * @param pTmpRsp Pointer to the temporary stack pointer.
10559 */
10560IEM_STATIC VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPU pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp)
10561{
10562 /* Increment the stack pointer. */
10563 RTUINT64U NewRsp = *pTmpRsp;
10564 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
10565
10566 /* Write the word the lazy way. */
10567 uint16_t const *pu16Src;
10568 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10569 if (rc == VINF_SUCCESS)
10570 {
10571 *pu16Value = *pu16Src;
10572 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
10573
10574 /* Commit the new RSP value. */
10575 if (rc == VINF_SUCCESS)
10576 *pTmpRsp = NewRsp;
10577 }
10578
10579 return rc;
10580}
10581
10582
10583/**
10584 * Pops a dword from the stack, using a temporary stack pointer.
10585 *
10586 * @returns Strict VBox status code.
10587 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10588 * @param pu32Value Where to store the popped value.
10589 * @param pTmpRsp Pointer to the temporary stack pointer.
10590 */
10591IEM_STATIC VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPU pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp)
10592{
10593 /* Increment the stack pointer. */
10594 RTUINT64U NewRsp = *pTmpRsp;
10595 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
10596
10597 /* Write the word the lazy way. */
10598 uint32_t const *pu32Src;
10599 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10600 if (rc == VINF_SUCCESS)
10601 {
10602 *pu32Value = *pu32Src;
10603 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
10604
10605 /* Commit the new RSP value. */
10606 if (rc == VINF_SUCCESS)
10607 *pTmpRsp = NewRsp;
10608 }
10609
10610 return rc;
10611}
10612
10613
10614/**
10615 * Pops a qword from the stack, using a temporary stack pointer.
10616 *
10617 * @returns Strict VBox status code.
10618 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10619 * @param pu64Value Where to store the popped value.
10620 * @param pTmpRsp Pointer to the temporary stack pointer.
10621 */
10622IEM_STATIC VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPU pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp)
10623{
10624 /* Increment the stack pointer. */
10625 RTUINT64U NewRsp = *pTmpRsp;
10626 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
10627
10628 /* Write the word the lazy way. */
10629 uint64_t const *pu64Src;
10630 VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10631 if (rcStrict == VINF_SUCCESS)
10632 {
10633 *pu64Value = *pu64Src;
10634 rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
10635
10636 /* Commit the new RSP value. */
10637 if (rcStrict == VINF_SUCCESS)
10638 *pTmpRsp = NewRsp;
10639 }
10640
10641 return rcStrict;
10642}
10643
10644
10645/**
10646 * Begin a special stack push (used by interrupt, exceptions and such).
10647 *
10648 * This will raise \#SS or \#PF if appropriate.
10649 *
10650 * @returns Strict VBox status code.
10651 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10652 * @param cbMem The number of bytes to push onto the stack.
10653 * @param ppvMem Where to return the pointer to the stack memory.
10654 * As with the other memory functions this could be
10655 * direct access or bounce buffered access, so
10656 * don't commit register until the commit call
10657 * succeeds.
10658 * @param puNewRsp Where to return the new RSP value. This must be
10659 * passed unchanged to
10660 * iemMemStackPushCommitSpecial().
10661 */
10662IEM_STATIC VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPU pVCpu, size_t cbMem, void **ppvMem, uint64_t *puNewRsp)
10663{
10664 Assert(cbMem < UINT8_MAX);
10665 RTGCPTR GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
10666 return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_W);
10667}
10668
10669
10670/**
10671 * Commits a special stack push (started by iemMemStackPushBeginSpecial).
10672 *
10673 * This will update the rSP.
10674 *
10675 * @returns Strict VBox status code.
10676 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10677 * @param pvMem The pointer returned by
10678 * iemMemStackPushBeginSpecial().
10679 * @param uNewRsp The new RSP value returned by
10680 * iemMemStackPushBeginSpecial().
10681 */
10682IEM_STATIC VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPU pVCpu, void *pvMem, uint64_t uNewRsp)
10683{
10684 VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
10685 if (rcStrict == VINF_SUCCESS)
10686 pVCpu->cpum.GstCtx.rsp = uNewRsp;
10687 return rcStrict;
10688}
10689
10690
10691/**
10692 * Begin a special stack pop (used by iret, retf and such).
10693 *
10694 * This will raise \#SS or \#PF if appropriate.
10695 *
10696 * @returns Strict VBox status code.
10697 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10698 * @param cbMem The number of bytes to pop from the stack.
10699 * @param ppvMem Where to return the pointer to the stack memory.
10700 * @param puNewRsp Where to return the new RSP value. This must be
10701 * assigned to CPUMCTX::rsp manually some time
10702 * after iemMemStackPopDoneSpecial() has been
10703 * called.
10704 */
10705IEM_STATIC VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPU pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
10706{
10707 Assert(cbMem < UINT8_MAX);
10708 RTGCPTR GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
10709 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10710}
10711
10712
10713/**
10714 * Continue a special stack pop (used by iret and retf).
10715 *
10716 * This will raise \#SS or \#PF if appropriate.
10717 *
10718 * @returns Strict VBox status code.
10719 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10720 * @param cbMem The number of bytes to pop from the stack.
10721 * @param ppvMem Where to return the pointer to the stack memory.
10722 * @param puNewRsp Where to return the new RSP value. This must be
10723 * assigned to CPUMCTX::rsp manually some time
10724 * after iemMemStackPopDoneSpecial() has been
10725 * called.
10726 */
10727IEM_STATIC VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPU pVCpu, size_t cbMem, void const **ppvMem, uint64_t *puNewRsp)
10728{
10729 Assert(cbMem < UINT8_MAX);
10730 RTUINT64U NewRsp;
10731 NewRsp.u = *puNewRsp;
10732 RTGCPTR GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
10733 *puNewRsp = NewRsp.u;
10734 return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R);
10735}
10736
10737
10738/**
10739 * Done with a special stack pop (started by iemMemStackPopBeginSpecial or
10740 * iemMemStackPopContinueSpecial).
10741 *
10742 * The caller will manually commit the rSP.
10743 *
10744 * @returns Strict VBox status code.
10745 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10746 * @param pvMem The pointer returned by
10747 * iemMemStackPopBeginSpecial() or
10748 * iemMemStackPopContinueSpecial().
10749 */
10750IEM_STATIC VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPU pVCpu, void const *pvMem)
10751{
10752 return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
10753}
10754
10755
10756/**
10757 * Fetches a system table byte.
10758 *
10759 * @returns Strict VBox status code.
10760 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10761 * @param pbDst Where to return the byte.
10762 * @param iSegReg The index of the segment register to use for
10763 * this access. The base and limits are checked.
10764 * @param GCPtrMem The address of the guest memory.
10765 */
10766IEM_STATIC VBOXSTRICTRC iemMemFetchSysU8(PVMCPU pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10767{
10768 /* The lazy approach for now... */
10769 uint8_t const *pbSrc;
10770 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10771 if (rc == VINF_SUCCESS)
10772 {
10773 *pbDst = *pbSrc;
10774 rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
10775 }
10776 return rc;
10777}
10778
10779
10780/**
10781 * Fetches a system table word.
10782 *
10783 * @returns Strict VBox status code.
10784 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10785 * @param pu16Dst Where to return the word.
10786 * @param iSegReg The index of the segment register to use for
10787 * this access. The base and limits are checked.
10788 * @param GCPtrMem The address of the guest memory.
10789 */
10790IEM_STATIC VBOXSTRICTRC iemMemFetchSysU16(PVMCPU pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10791{
10792 /* The lazy approach for now... */
10793 uint16_t const *pu16Src;
10794 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10795 if (rc == VINF_SUCCESS)
10796 {
10797 *pu16Dst = *pu16Src;
10798 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
10799 }
10800 return rc;
10801}
10802
10803
10804/**
10805 * Fetches a system table dword.
10806 *
10807 * @returns Strict VBox status code.
10808 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10809 * @param pu32Dst Where to return the dword.
10810 * @param iSegReg The index of the segment register to use for
10811 * this access. The base and limits are checked.
10812 * @param GCPtrMem The address of the guest memory.
10813 */
10814IEM_STATIC VBOXSTRICTRC iemMemFetchSysU32(PVMCPU pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10815{
10816 /* The lazy approach for now... */
10817 uint32_t const *pu32Src;
10818 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10819 if (rc == VINF_SUCCESS)
10820 {
10821 *pu32Dst = *pu32Src;
10822 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
10823 }
10824 return rc;
10825}
10826
10827
10828/**
10829 * Fetches a system table qword.
10830 *
10831 * @returns Strict VBox status code.
10832 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10833 * @param pu64Dst Where to return the qword.
10834 * @param iSegReg The index of the segment register to use for
10835 * this access. The base and limits are checked.
10836 * @param GCPtrMem The address of the guest memory.
10837 */
10838IEM_STATIC VBOXSTRICTRC iemMemFetchSysU64(PVMCPU pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem)
10839{
10840 /* The lazy approach for now... */
10841 uint64_t const *pu64Src;
10842 VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R);
10843 if (rc == VINF_SUCCESS)
10844 {
10845 *pu64Dst = *pu64Src;
10846 rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
10847 }
10848 return rc;
10849}
10850
10851
10852/**
10853 * Fetches a descriptor table entry with caller specified error code.
10854 *
10855 * @returns Strict VBox status code.
10856 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10857 * @param pDesc Where to return the descriptor table entry.
10858 * @param uSel The selector which table entry to fetch.
10859 * @param uXcpt The exception to raise on table lookup error.
10860 * @param uErrorCode The error code associated with the exception.
10861 */
10862IEM_STATIC VBOXSTRICTRC
10863iemMemFetchSelDescWithErr(PVMCPU pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt, uint16_t uErrorCode)
10864{
10865 AssertPtr(pDesc);
10866 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
10867
10868 /** @todo did the 286 require all 8 bytes to be accessible? */
10869 /*
10870 * Get the selector table base and check bounds.
10871 */
10872 RTGCPTR GCPtrBase;
10873 if (uSel & X86_SEL_LDT)
10874 {
10875 if ( !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
10876 || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
10877 {
10878 Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
10879 uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
10880 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
10881 uErrorCode, 0);
10882 }
10883
10884 Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
10885 GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
10886 }
10887 else
10888 {
10889 if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
10890 {
10891 Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
10892 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
10893 uErrorCode, 0);
10894 }
10895 GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
10896 }
10897
10898 /*
10899 * Read the legacy descriptor and maybe the long mode extensions if
10900 * required.
10901 */
10902 VBOXSTRICTRC rcStrict;
10903 if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
10904 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
10905 else
10906 {
10907 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
10908 if (rcStrict == VINF_SUCCESS)
10909 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
10910 if (rcStrict == VINF_SUCCESS)
10911 rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
10912 if (rcStrict == VINF_SUCCESS)
10913 pDesc->Legacy.au16[3] = 0;
10914 else
10915 return rcStrict;
10916 }
10917
10918 if (rcStrict == VINF_SUCCESS)
10919 {
10920 if ( !IEM_IS_LONG_MODE(pVCpu)
10921 || pDesc->Legacy.Gen.u1DescType)
10922 pDesc->Long.au64[1] = 0;
10923 else if ((uint32_t)(uSel | X86_SEL_RPL_LDT) + 8 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
10924 rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1);
10925 else
10926 {
10927 Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
10928 /** @todo is this the right exception? */
10929 return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
10930 }
10931 }
10932 return rcStrict;
10933}
10934
10935
10936/**
10937 * Fetches a descriptor table entry.
10938 *
10939 * @returns Strict VBox status code.
10940 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10941 * @param pDesc Where to return the descriptor table entry.
10942 * @param uSel The selector which table entry to fetch.
10943 * @param uXcpt The exception to raise on table lookup error.
10944 */
10945IEM_STATIC VBOXSTRICTRC iemMemFetchSelDesc(PVMCPU pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt)
10946{
10947 return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
10948}
10949
10950
10951/**
10952 * Fakes a long mode stack selector for SS = 0.
10953 *
10954 * @param pDescSs Where to return the fake stack descriptor.
10955 * @param uDpl The DPL we want.
10956 */
10957IEM_STATIC void iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
10958{
10959 pDescSs->Long.au64[0] = 0;
10960 pDescSs->Long.au64[1] = 0;
10961 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
10962 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
10963 pDescSs->Long.Gen.u2Dpl = uDpl;
10964 pDescSs->Long.Gen.u1Present = 1;
10965 pDescSs->Long.Gen.u1Long = 1;
10966}
10967
10968
10969/**
10970 * Marks the selector descriptor as accessed (only non-system descriptors).
10971 *
10972 * This function ASSUMES that iemMemFetchSelDesc has be called previously and
10973 * will therefore skip the limit checks.
10974 *
10975 * @returns Strict VBox status code.
10976 * @param pVCpu The cross context virtual CPU structure of the calling thread.
10977 * @param uSel The selector.
10978 */
10979IEM_STATIC VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPU pVCpu, uint16_t uSel)
10980{
10981 /*
10982 * Get the selector table base and calculate the entry address.
10983 */
10984 RTGCPTR GCPtr = uSel & X86_SEL_LDT
10985 ? pVCpu->cpum.GstCtx.ldtr.u64Base
10986 : pVCpu->cpum.GstCtx.gdtr.pGdt;
10987 GCPtr += uSel & X86_SEL_MASK;
10988
10989 /*
10990 * ASMAtomicBitSet will assert if the address is misaligned, so do some
10991 * ugly stuff to avoid this. This will make sure it's an atomic access
10992 * as well more or less remove any question about 8-bit or 32-bit accesss.
10993 */
10994 VBOXSTRICTRC rcStrict;
10995 uint32_t volatile *pu32;
10996 if ((GCPtr & 3) == 0)
10997 {
10998 /* The normal case, map the 32-bit bits around the accessed bit (40). */
10999 GCPtr += 2 + 2;
11000 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
11001 if (rcStrict != VINF_SUCCESS)
11002 return rcStrict;
11003 ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
11004 }
11005 else
11006 {
11007 /* The misaligned GDT/LDT case, map the whole thing. */
11008 rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW);
11009 if (rcStrict != VINF_SUCCESS)
11010 return rcStrict;
11011 switch ((uintptr_t)pu32 & 3)
11012 {
11013 case 0: ASMAtomicBitSet(pu32, 40 + 0 - 0); break;
11014 case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
11015 case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
11016 case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 - 8); break;
11017 }
11018 }
11019
11020 return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
11021}
11022
11023/** @} */
11024
11025
11026/*
11027 * Include the C/C++ implementation of instruction.
11028 */
11029#include "IEMAllCImpl.cpp.h"
11030
11031
11032
11033/** @name "Microcode" macros.
11034 *
11035 * The idea is that we should be able to use the same code to interpret
11036 * instructions as well as recompiler instructions. Thus this obfuscation.
11037 *
11038 * @{
11039 */
11040#define IEM_MC_BEGIN(a_cArgs, a_cLocals) {
11041#define IEM_MC_END() }
11042#define IEM_MC_PAUSE() do {} while (0)
11043#define IEM_MC_CONTINUE() do {} while (0)
11044
11045/** Internal macro. */
11046#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
11047 do \
11048 { \
11049 VBOXSTRICTRC rcStrict2 = a_Expr; \
11050 if (rcStrict2 != VINF_SUCCESS) \
11051 return rcStrict2; \
11052 } while (0)
11053
11054
11055#define IEM_MC_ADVANCE_RIP() iemRegUpdateRipAndClearRF(pVCpu)
11056#define IEM_MC_REL_JMP_S8(a_i8) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS8(pVCpu, a_i8))
11057#define IEM_MC_REL_JMP_S16(a_i16) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS16(pVCpu, a_i16))
11058#define IEM_MC_REL_JMP_S32(a_i32) IEM_MC_RETURN_ON_FAILURE(iemRegRipRelativeJumpS32(pVCpu, a_i32))
11059#define IEM_MC_SET_RIP_U16(a_u16NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u16NewIP)))
11060#define IEM_MC_SET_RIP_U32(a_u32NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u32NewIP)))
11061#define IEM_MC_SET_RIP_U64(a_u64NewIP) IEM_MC_RETURN_ON_FAILURE(iemRegRipJump((pVCpu), (a_u64NewIP)))
11062#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
11063#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
11064 do { \
11065 if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS)) \
11066 return iemRaiseDeviceNotAvailable(pVCpu); \
11067 } while (0)
11068#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
11069 do { \
11070 if ((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP | X86_CR0_TS)) == (X86_CR0_MP | X86_CR0_TS)) \
11071 return iemRaiseDeviceNotAvailable(pVCpu); \
11072 } while (0)
11073#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
11074 do { \
11075 if (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW & X86_FSW_ES) \
11076 return iemRaiseMathFault(pVCpu); \
11077 } while (0)
11078#define IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT() \
11079 do { \
11080 if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
11081 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
11082 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx2) \
11083 return iemRaiseUndefinedOpcode(pVCpu); \
11084 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11085 return iemRaiseDeviceNotAvailable(pVCpu); \
11086 } while (0)
11087#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
11088 do { \
11089 if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM | XSAVE_C_SSE)) != (XSAVE_C_YMM | XSAVE_C_SSE) \
11090 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
11091 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx) \
11092 return iemRaiseUndefinedOpcode(pVCpu); \
11093 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11094 return iemRaiseDeviceNotAvailable(pVCpu); \
11095 } while (0)
11096#define IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT() \
11097 do { \
11098 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11099 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11100 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse41) \
11101 return iemRaiseUndefinedOpcode(pVCpu); \
11102 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11103 return iemRaiseDeviceNotAvailable(pVCpu); \
11104 } while (0)
11105#define IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT() \
11106 do { \
11107 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11108 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11109 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse3) \
11110 return iemRaiseUndefinedOpcode(pVCpu); \
11111 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11112 return iemRaiseDeviceNotAvailable(pVCpu); \
11113 } while (0)
11114#define IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT() \
11115 do { \
11116 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11117 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11118 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2) \
11119 return iemRaiseUndefinedOpcode(pVCpu); \
11120 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11121 return iemRaiseDeviceNotAvailable(pVCpu); \
11122 } while (0)
11123#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
11124 do { \
11125 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11126 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR) \
11127 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse) \
11128 return iemRaiseUndefinedOpcode(pVCpu); \
11129 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11130 return iemRaiseDeviceNotAvailable(pVCpu); \
11131 } while (0)
11132#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
11133 do { \
11134 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11135 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMmx) \
11136 return iemRaiseUndefinedOpcode(pVCpu); \
11137 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11138 return iemRaiseDeviceNotAvailable(pVCpu); \
11139 } while (0)
11140#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT() \
11141 do { \
11142 if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
11143 || ( !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse \
11144 && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAmdMmxExts) ) \
11145 return iemRaiseUndefinedOpcode(pVCpu); \
11146 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
11147 return iemRaiseDeviceNotAvailable(pVCpu); \
11148 } while (0)
11149#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
11150 do { \
11151 if (pVCpu->iem.s.uCpl != 0) \
11152 return iemRaiseGeneralProtectionFault0(pVCpu); \
11153 } while (0)
11154#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
11155 do { \
11156 if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
11157 else return iemRaiseGeneralProtectionFault0(pVCpu); \
11158 } while (0)
11159#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
11160 do { \
11161 if ( pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT \
11162 || !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFsGsBase \
11163 || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE)) \
11164 return iemRaiseUndefinedOpcode(pVCpu); \
11165 } while (0)
11166#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
11167 do { \
11168 if (!IEM_IS_CANONICAL(a_u64Addr)) \
11169 return iemRaiseGeneralProtectionFault0(pVCpu); \
11170 } while (0)
11171
11172
11173#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
11174#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
11175#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
11176#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
11177#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
11178#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
11179#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
11180 uint32_t a_Name; \
11181 uint32_t *a_pName = &a_Name
11182#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
11183 do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
11184
11185#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
11186#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
11187
11188#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11189#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11190#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11191#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
11192#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11193#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11194#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
11195#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11196#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11197#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
11198#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
11199#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
11200#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
11201#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
11202#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
11203#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
11204#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
11205#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
11206 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11207 (a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11208 } while (0)
11209#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
11210 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11211 (a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11212 } while (0)
11213#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
11214 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11215 (a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
11216 } while (0)
11217/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
11218#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
11219 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11220 (a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
11221 } while (0)
11222#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
11223 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11224 (a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
11225 } while (0)
11226/** @note Not for IOPL or IF testing or modification. */
11227#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
11228#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
11229#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW
11230#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW
11231
11232#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
11233#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
11234#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) /* clear high bits. */
11235#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
11236#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
11237#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
11238#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
11239#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
11240#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
11241#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
11242/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
11243#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
11244 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11245 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
11246 } while (0)
11247#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
11248 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
11249 *iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); /* clear high bits. */ \
11250 } while (0)
11251#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
11252 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
11253
11254
11255#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
11256#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
11257/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
11258 * Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
11259#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
11260#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
11261/** @note Not for IOPL or IF testing or modification. */
11262#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.u
11263
11264#define IEM_MC_ADD_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) += (a_u8Value)
11265#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
11266#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
11267 do { \
11268 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11269 *pu32Reg += (a_u32Value); \
11270 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11271 } while (0)
11272#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
11273
11274#define IEM_MC_SUB_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) -= (a_u8Value)
11275#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u16Value)
11276#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u32Value) \
11277 do { \
11278 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11279 *pu32Reg -= (a_u32Value); \
11280 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11281 } while (0)
11282#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u64Value)
11283#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
11284
11285#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
11286#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
11287#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
11288#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
11289#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
11290#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
11291#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
11292
11293#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
11294#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
11295#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
11296#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
11297
11298#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
11299#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
11300#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
11301
11302#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) |= (a_u8Mask); } while (0)
11303#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) |= (a_u16Mask); } while (0)
11304#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
11305
11306#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
11307#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
11308#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
11309
11310#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
11311#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
11312#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
11313
11314#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
11315
11316#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) |= (a_u32Mask); } while (0)
11317
11318#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
11319#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
11320#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
11321 do { \
11322 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11323 *pu32Reg &= (a_u32Value); \
11324 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11325 } while (0)
11326#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
11327
11328#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) |= (a_u8Value)
11329#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) |= (a_u16Value)
11330#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
11331 do { \
11332 uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
11333 *pu32Reg |= (a_u32Value); \
11334 pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
11335 } while (0)
11336#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) |= (a_u64Value)
11337
11338
11339/** @note Not for IOPL or IF modification. */
11340#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u |= (a_fBit); } while (0)
11341/** @note Not for IOPL or IF modification. */
11342#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
11343/** @note Not for IOPL or IF modification. */
11344#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
11345
11346#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW &= X86_FSW_C_MASK | X86_FSW_TOP_MASK; } while (0)
11347
11348/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
11349#define IEM_MC_FPU_TO_MMX_MODE() do { \
11350 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FSW &= ~X86_FSW_TOP_MASK; \
11351 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FTW = 0xff; \
11352 } while (0)
11353
11354/** Switches the FPU state from MMX mode (FTW=0xffff). */
11355#define IEM_MC_FPU_FROM_MMX_MODE() do { \
11356 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FTW = 0; \
11357 } while (0)
11358
11359#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
11360 do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx; } while (0)
11361#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
11362 do { (a_u32Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[0]; } while (0)
11363#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
11364 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
11365 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
11366 } while (0)
11367#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
11368 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
11369 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
11370 } while (0)
11371#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
11372 (a_pu64Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
11373#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
11374 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
11375#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
11376 (a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aRegs[(a_iMReg)].mmx)
11377
11378#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
11379 do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0]; \
11380 (a_u128Value).au64[1] = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1]; \
11381 } while (0)
11382#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg) \
11383 do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0]; } while (0)
11384#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg) \
11385 do { (a_u32Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au32[0]; } while (0)
11386#define IEM_MC_FETCH_XREG_HI_U64(a_u64Value, a_iXReg) \
11387 do { (a_u64Value) = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1]; } while (0)
11388#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
11389 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
11390 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
11391 } while (0)
11392#define IEM_MC_STORE_XREG_U64(a_iXReg, a_u64Value) \
11393 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); } while (0)
11394#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
11395 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
11396 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = 0; \
11397 } while (0)
11398#define IEM_MC_STORE_XREG_U32(a_iXReg, a_u32Value) \
11399 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au32[0] = (a_u32Value); } while (0)
11400#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
11401 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
11402 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = 0; \
11403 } while (0)
11404#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
11405 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
11406#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
11407 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].uXmm)
11408#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
11409 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].uXmm)
11410#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
11411 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXReg)].au64[0])
11412#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
11413 do { pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegDst)].au64[0] \
11414 = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegSrc)].au64[0]; \
11415 pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegDst)].au64[1] \
11416 = pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aXMM[(a_iXRegSrc)].au64[1]; \
11417 } while (0)
11418
11419#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
11420 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11421 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11422 (a_u32Dst) = pXStateTmp->x87.aXMM[iYRegSrcTmp].au32[0]; \
11423 } while (0)
11424#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
11425 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11426 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11427 (a_u64Dst) = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11428 } while (0)
11429#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
11430 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11431 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11432 (a_u128Dst).au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11433 (a_u128Dst).au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11434 } while (0)
11435#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
11436 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11437 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11438 (a_u256Dst).au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11439 (a_u256Dst).au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11440 (a_u256Dst).au64[2] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
11441 (a_u256Dst).au64[3] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
11442 } while (0)
11443
11444#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_pXState, a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
11445#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
11446 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11447 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11448 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
11449 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[1] = 0; \
11450 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11451 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11452 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11453 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11454 } while (0)
11455#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
11456 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11457 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11458 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
11459 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11460 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11461 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11462 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11463 } while (0)
11464#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
11465 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11466 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11467 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
11468 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
11469 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11470 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11471 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11472 } while (0)
11473#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
11474 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11475 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11476 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
11477 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
11478 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
11479 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
11480 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11481 } while (0)
11482
11483#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
11484 (a_pu128Dst) = (&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].uXmm)
11485#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
11486 (a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].uXmm)
11487#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
11488 (a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.aYMM[(a_iYReg)].au64[0])
11489#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
11490 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11491 uintptr_t const iYRegTmp = (a_iYReg); \
11492 pXStateTmp->u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
11493 pXStateTmp->u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
11494 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegTmp); \
11495 } while (0)
11496
11497#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11498 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11499 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11500 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11501 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11502 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11503 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
11504 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pXStateTmp->u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
11505 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11506 } while (0)
11507#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11508 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11509 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11510 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11511 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11512 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[1]; \
11513 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11514 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11515 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11516 } while (0)
11517#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
11518 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11519 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11520 uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
11521 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrcTmp].au64[0]; \
11522 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = 0; \
11523 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11524 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11525 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11526 } while (0)
11527
11528#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
11529 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11530 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11531 uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
11532 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11533 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[0] = pXStateTmp->x87.aXMM[iYRegSrc32Tmp].au32[0]; \
11534 pXStateTmp->x87.aXMM[iYRegDstTmp].au32[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au32[1]; \
11535 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11536 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11537 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11538 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11539 } while (0)
11540#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
11541 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11542 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11543 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
11544 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11545 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrc64Tmp].au64[0]; \
11546 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11547 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11548 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11549 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11550 } while (0)
11551#define IEM_MC_MERGE_YREG_U64HI_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
11552 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11553 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11554 uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
11555 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11556 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = pXStateTmp->x87.aXMM[iYRegSrc64Tmp].au64[1]; \
11557 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11558 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11559 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11560 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11561 } while (0)
11562#define IEM_MC_MERGE_YREG_U64LOCAL_U64_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
11563 do { PX86XSAVEAREA pXStateTmp = pVCpu->cpum.GstCtx.CTX_SUFF(pXState); \
11564 uintptr_t const iYRegDstTmp = (a_iYRegDst); \
11565 uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
11566 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
11567 pXStateTmp->x87.aXMM[iYRegDstTmp].au64[1] = pXStateTmp->x87.aXMM[iYRegSrcHxTmp].au64[1]; \
11568 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
11569 pXStateTmp->u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
11570 IEM_MC_INT_CLEAR_ZMM_256_UP(pXStateTmp, iYRegDstTmp); \
11571 } while (0)
11572
11573#ifndef IEM_WITH_SETJMP
11574# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
11575 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
11576# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
11577 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
11578# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
11579 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
11580#else
11581# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
11582 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11583# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
11584 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
11585# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
11586 ((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
11587#endif
11588
11589#ifndef IEM_WITH_SETJMP
11590# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11591 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
11592# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11593 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11594# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
11595 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
11596#else
11597# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11598 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11599# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11600 ((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11601# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
11602 ((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11603#endif
11604
11605#ifndef IEM_WITH_SETJMP
11606# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11607 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
11608# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11609 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11610# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
11611 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
11612#else
11613# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11614 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11615# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11616 ((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11617# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
11618 ((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11619#endif
11620
11621#ifdef SOME_UNUSED_FUNCTION
11622# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11623 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11624#endif
11625
11626#ifndef IEM_WITH_SETJMP
11627# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11628 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11629# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11630 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11631# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
11632 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
11633# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
11634 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
11635#else
11636# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11637 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11638# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
11639 ((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
11640# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
11641 ((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11642# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
11643 ((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11644#endif
11645
11646#ifndef IEM_WITH_SETJMP
11647# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
11648 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u32, (a_iSeg), (a_GCPtrMem)))
11649# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
11650 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).au64[0], (a_iSeg), (a_GCPtrMem)))
11651# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
11652 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
11653#else
11654# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
11655 ((a_r32Dst).u32 = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11656# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
11657 ((a_r64Dst).au64[0] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11658# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
11659 iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
11660#endif
11661
11662#ifndef IEM_WITH_SETJMP
11663# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
11664 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
11665# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
11666 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
11667#else
11668# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
11669 iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
11670# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
11671 iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
11672#endif
11673
11674#ifndef IEM_WITH_SETJMP
11675# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
11676 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
11677# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
11678 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
11679#else
11680# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
11681 iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
11682# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
11683 iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
11684#endif
11685
11686
11687
11688#ifndef IEM_WITH_SETJMP
11689# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11690 do { \
11691 uint8_t u8Tmp; \
11692 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11693 (a_u16Dst) = u8Tmp; \
11694 } while (0)
11695# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11696 do { \
11697 uint8_t u8Tmp; \
11698 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11699 (a_u32Dst) = u8Tmp; \
11700 } while (0)
11701# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11702 do { \
11703 uint8_t u8Tmp; \
11704 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11705 (a_u64Dst) = u8Tmp; \
11706 } while (0)
11707# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11708 do { \
11709 uint16_t u16Tmp; \
11710 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11711 (a_u32Dst) = u16Tmp; \
11712 } while (0)
11713# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11714 do { \
11715 uint16_t u16Tmp; \
11716 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11717 (a_u64Dst) = u16Tmp; \
11718 } while (0)
11719# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11720 do { \
11721 uint32_t u32Tmp; \
11722 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
11723 (a_u64Dst) = u32Tmp; \
11724 } while (0)
11725#else /* IEM_WITH_SETJMP */
11726# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11727 ((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11728# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11729 ((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11730# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11731 ((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11732# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11733 ((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11734# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11735 ((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11736# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11737 ((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11738#endif /* IEM_WITH_SETJMP */
11739
11740#ifndef IEM_WITH_SETJMP
11741# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11742 do { \
11743 uint8_t u8Tmp; \
11744 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11745 (a_u16Dst) = (int8_t)u8Tmp; \
11746 } while (0)
11747# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11748 do { \
11749 uint8_t u8Tmp; \
11750 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11751 (a_u32Dst) = (int8_t)u8Tmp; \
11752 } while (0)
11753# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11754 do { \
11755 uint8_t u8Tmp; \
11756 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
11757 (a_u64Dst) = (int8_t)u8Tmp; \
11758 } while (0)
11759# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11760 do { \
11761 uint16_t u16Tmp; \
11762 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11763 (a_u32Dst) = (int16_t)u16Tmp; \
11764 } while (0)
11765# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11766 do { \
11767 uint16_t u16Tmp; \
11768 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
11769 (a_u64Dst) = (int16_t)u16Tmp; \
11770 } while (0)
11771# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11772 do { \
11773 uint32_t u32Tmp; \
11774 IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
11775 (a_u64Dst) = (int32_t)u32Tmp; \
11776 } while (0)
11777#else /* IEM_WITH_SETJMP */
11778# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
11779 ((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11780# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11781 ((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11782# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11783 ((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11784# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
11785 ((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11786# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11787 ((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11788# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
11789 ((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
11790#endif /* IEM_WITH_SETJMP */
11791
11792#ifndef IEM_WITH_SETJMP
11793# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
11794 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
11795# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
11796 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
11797# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
11798 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
11799# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
11800 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
11801#else
11802# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
11803 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
11804# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
11805 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
11806# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
11807 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
11808# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
11809 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
11810#endif
11811
11812#ifndef IEM_WITH_SETJMP
11813# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
11814 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
11815# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
11816 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
11817# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
11818 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
11819# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
11820 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
11821#else
11822# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
11823 iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
11824# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
11825 iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
11826# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
11827 iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
11828# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
11829 iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
11830#endif
11831
11832#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
11833#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
11834#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
11835#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
11836#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u32 = UINT32_C(0xffc00000)
11837#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->au64[0] = UINT64_C(0xfff8000000000000)
11838#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
11839 do { \
11840 (a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
11841 (a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
11842 } while (0)
11843
11844#ifndef IEM_WITH_SETJMP
11845# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
11846 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
11847# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
11848 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
11849#else
11850# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
11851 iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
11852# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
11853 iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
11854#endif
11855
11856#ifndef IEM_WITH_SETJMP
11857# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
11858 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
11859# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
11860 IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
11861#else
11862# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
11863 iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
11864# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
11865 iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
11866#endif
11867
11868
11869#define IEM_MC_PUSH_U16(a_u16Value) \
11870 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
11871#define IEM_MC_PUSH_U32(a_u32Value) \
11872 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
11873#define IEM_MC_PUSH_U32_SREG(a_u32Value) \
11874 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_u32Value)))
11875#define IEM_MC_PUSH_U64(a_u64Value) \
11876 IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
11877
11878#define IEM_MC_POP_U16(a_pu16Value) \
11879 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
11880#define IEM_MC_POP_U32(a_pu32Value) \
11881 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
11882#define IEM_MC_POP_U64(a_pu64Value) \
11883 IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
11884
11885/** Maps guest memory for direct or bounce buffered access.
11886 * The purpose is to pass it to an operand implementation, thus the a_iArg.
11887 * @remarks May return.
11888 */
11889#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
11890 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pMem), sizeof(*(a_pMem)), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
11891
11892/** Maps guest memory for direct or bounce buffered access.
11893 * The purpose is to pass it to an operand implementation, thus the a_iArg.
11894 * @remarks May return.
11895 */
11896#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_iArg) \
11897 IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), (a_GCPtrMem), (a_fAccess)))
11898
11899/** Commits the memory and unmaps the guest memory.
11900 * @remarks May return.
11901 */
11902#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
11903 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
11904
11905/** Commits the memory and unmaps the guest memory unless the FPU status word
11906 * indicates (@a a_u16FSW) and FPU control word indicates a pending exception
11907 * that would cause FLD not to store.
11908 *
11909 * The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
11910 * store, while \#P will not.
11911 *
11912 * @remarks May in theory return - for now.
11913 */
11914#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
11915 do { \
11916 if ( !(a_u16FSW & X86_FSW_ES) \
11917 || !( (a_u16FSW & (X86_FSW_UE | X86_FSW_OE | X86_FSW_IE)) \
11918 & ~(pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW & X86_FCW_MASK_ALL) ) ) \
11919 IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
11920 } while (0)
11921
11922/** Calculate efficient address from R/M. */
11923#ifndef IEM_WITH_SETJMP
11924# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
11925 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (bRm), (cbImm), &(a_GCPtrEff)))
11926#else
11927# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, bRm, cbImm) \
11928 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (bRm), (cbImm)))
11929#endif
11930
11931#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
11932#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
11933#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
11934#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
11935#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
11936#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
11937#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
11938
11939/**
11940 * Defers the rest of the instruction emulation to a C implementation routine
11941 * and returns, only taking the standard parameters.
11942 *
11943 * @param a_pfnCImpl The pointer to the C routine.
11944 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
11945 */
11946#define IEM_MC_CALL_CIMPL_0(a_pfnCImpl) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
11947
11948/**
11949 * Defers the rest of instruction emulation to a C implementation routine and
11950 * returns, taking one argument in addition to the standard ones.
11951 *
11952 * @param a_pfnCImpl The pointer to the C routine.
11953 * @param a0 The argument.
11954 */
11955#define IEM_MC_CALL_CIMPL_1(a_pfnCImpl, a0) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
11956
11957/**
11958 * Defers the rest of the instruction emulation to a C implementation routine
11959 * and returns, taking two arguments in addition to the standard ones.
11960 *
11961 * @param a_pfnCImpl The pointer to the C routine.
11962 * @param a0 The first extra argument.
11963 * @param a1 The second extra argument.
11964 */
11965#define IEM_MC_CALL_CIMPL_2(a_pfnCImpl, a0, a1) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
11966
11967/**
11968 * Defers the rest of the instruction emulation to a C implementation routine
11969 * and returns, taking three arguments in addition to the standard ones.
11970 *
11971 * @param a_pfnCImpl The pointer to the C routine.
11972 * @param a0 The first extra argument.
11973 * @param a1 The second extra argument.
11974 * @param a2 The third extra argument.
11975 */
11976#define IEM_MC_CALL_CIMPL_3(a_pfnCImpl, a0, a1, a2) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
11977
11978/**
11979 * Defers the rest of the instruction emulation to a C implementation routine
11980 * and returns, taking four arguments in addition to the standard ones.
11981 *
11982 * @param a_pfnCImpl The pointer to the C routine.
11983 * @param a0 The first extra argument.
11984 * @param a1 The second extra argument.
11985 * @param a2 The third extra argument.
11986 * @param a3 The fourth extra argument.
11987 */
11988#define IEM_MC_CALL_CIMPL_4(a_pfnCImpl, a0, a1, a2, a3) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3)
11989
11990/**
11991 * Defers the rest of the instruction emulation to a C implementation routine
11992 * and returns, taking two arguments in addition to the standard ones.
11993 *
11994 * @param a_pfnCImpl The pointer to the C routine.
11995 * @param a0 The first extra argument.
11996 * @param a1 The second extra argument.
11997 * @param a2 The third extra argument.
11998 * @param a3 The fourth extra argument.
11999 * @param a4 The fifth extra argument.
12000 */
12001#define IEM_MC_CALL_CIMPL_5(a_pfnCImpl, a0, a1, a2, a3, a4) return (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4)
12002
12003/**
12004 * Defers the entire instruction emulation to a C implementation routine and
12005 * returns, only taking the standard parameters.
12006 *
12007 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12008 *
12009 * @param a_pfnCImpl The pointer to the C routine.
12010 * @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
12011 */
12012#define IEM_MC_DEFER_TO_CIMPL_0(a_pfnCImpl) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
12013
12014/**
12015 * Defers the entire instruction emulation to a C implementation routine and
12016 * returns, taking one argument in addition to the standard ones.
12017 *
12018 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12019 *
12020 * @param a_pfnCImpl The pointer to the C routine.
12021 * @param a0 The argument.
12022 */
12023#define IEM_MC_DEFER_TO_CIMPL_1(a_pfnCImpl, a0) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0)
12024
12025/**
12026 * Defers the entire instruction emulation to a C implementation routine and
12027 * returns, taking two arguments in addition to the standard ones.
12028 *
12029 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12030 *
12031 * @param a_pfnCImpl The pointer to the C routine.
12032 * @param a0 The first extra argument.
12033 * @param a1 The second extra argument.
12034 */
12035#define IEM_MC_DEFER_TO_CIMPL_2(a_pfnCImpl, a0, a1) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1)
12036
12037/**
12038 * Defers the entire instruction emulation to a C implementation routine and
12039 * returns, taking three arguments in addition to the standard ones.
12040 *
12041 * This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
12042 *
12043 * @param a_pfnCImpl The pointer to the C routine.
12044 * @param a0 The first extra argument.
12045 * @param a1 The second extra argument.
12046 * @param a2 The third extra argument.
12047 */
12048#define IEM_MC_DEFER_TO_CIMPL_3(a_pfnCImpl, a0, a1, a2) (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2)
12049
12050/**
12051 * Calls a FPU assembly implementation taking one visible argument.
12052 *
12053 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12054 * @param a0 The first extra argument.
12055 */
12056#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
12057 do { \
12058 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0)); \
12059 } while (0)
12060
12061/**
12062 * Calls a FPU assembly implementation taking two visible arguments.
12063 *
12064 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12065 * @param a0 The first extra argument.
12066 * @param a1 The second extra argument.
12067 */
12068#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
12069 do { \
12070 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
12071 } while (0)
12072
12073/**
12074 * Calls a FPU assembly implementation taking three visible arguments.
12075 *
12076 * @param a_pfnAImpl Pointer to the assembly FPU routine.
12077 * @param a0 The first extra argument.
12078 * @param a1 The second extra argument.
12079 * @param a2 The third extra argument.
12080 */
12081#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12082 do { \
12083 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
12084 } while (0)
12085
12086#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
12087 do { \
12088 (a_FpuData).FSW = (a_FSW); \
12089 (a_FpuData).r80Result = *(a_pr80Value); \
12090 } while (0)
12091
12092/** Pushes FPU result onto the stack. */
12093#define IEM_MC_PUSH_FPU_RESULT(a_FpuData) \
12094 iemFpuPushResult(pVCpu, &a_FpuData)
12095/** Pushes FPU result onto the stack and sets the FPUDP. */
12096#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff) \
12097 iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff)
12098
12099/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
12100#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo) \
12101 iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo)
12102
12103/** Stores FPU result in a stack register. */
12104#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg) \
12105 iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg)
12106/** Stores FPU result in a stack register and pops the stack. */
12107#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg) \
12108 iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg)
12109/** Stores FPU result in a stack register and sets the FPUDP. */
12110#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
12111 iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
12112/** Stores FPU result in a stack register, sets the FPUDP, and pops the
12113 * stack. */
12114#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff) \
12115 iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff)
12116
12117/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
12118#define IEM_MC_UPDATE_FPU_OPCODE_IP() \
12119 iemFpuUpdateOpcodeAndIp(pVCpu)
12120/** Free a stack register (for FFREE and FFREEP). */
12121#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
12122 iemFpuStackFree(pVCpu, a_iStReg)
12123/** Increment the FPU stack pointer. */
12124#define IEM_MC_FPU_STACK_INC_TOP() \
12125 iemFpuStackIncTop(pVCpu)
12126/** Decrement the FPU stack pointer. */
12127#define IEM_MC_FPU_STACK_DEC_TOP() \
12128 iemFpuStackDecTop(pVCpu)
12129
12130/** Updates the FSW, FOP, FPUIP, and FPUCS. */
12131#define IEM_MC_UPDATE_FSW(a_u16FSW) \
12132 iemFpuUpdateFSW(pVCpu, a_u16FSW)
12133/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
12134#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW) \
12135 iemFpuUpdateFSW(pVCpu, a_u16FSW)
12136/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
12137#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
12138 iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
12139/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
12140#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW) \
12141 iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW)
12142/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
12143 * stack. */
12144#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff) \
12145 iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff)
12146/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
12147#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW) \
12148 iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW)
12149
12150/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
12151#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst) \
12152 iemFpuStackUnderflow(pVCpu, a_iStDst)
12153/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
12154 * stack. */
12155#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst) \
12156 iemFpuStackUnderflowThenPop(pVCpu, a_iStDst)
12157/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
12158 * FPUDS. */
12159#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
12160 iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
12161/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
12162 * FPUDS. Pops stack. */
12163#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff) \
12164 iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff)
12165/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
12166 * stack twice. */
12167#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP() \
12168 iemFpuStackUnderflowThenPopPop(pVCpu)
12169/** Raises a FPU stack underflow exception for an instruction pushing a result
12170 * value onto the stack. Sets FPUIP, FPUCS and FOP. */
12171#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW() \
12172 iemFpuStackPushUnderflow(pVCpu)
12173/** Raises a FPU stack underflow exception for an instruction pushing a result
12174 * value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
12175#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO() \
12176 iemFpuStackPushUnderflowTwo(pVCpu)
12177
12178/** Raises a FPU stack overflow exception as part of a push attempt. Sets
12179 * FPUIP, FPUCS and FOP. */
12180#define IEM_MC_FPU_STACK_PUSH_OVERFLOW() \
12181 iemFpuStackPushOverflow(pVCpu)
12182/** Raises a FPU stack overflow exception as part of a push attempt. Sets
12183 * FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
12184#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff) \
12185 iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff)
12186/** Prepares for using the FPU state.
12187 * Ensures that we can use the host FPU in the current context (RC+R0.
12188 * Ensures the guest FPU state in the CPUMCTX is up to date. */
12189#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
12190/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
12191#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
12192/** Actualizes the guest FPU state so it can be accessed and modified. */
12193#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
12194
12195/** Prepares for using the SSE state.
12196 * Ensures that we can use the host SSE/FPU in the current context (RC+R0.
12197 * Ensures the guest SSE state in the CPUMCTX is up to date. */
12198#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
12199/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
12200#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
12201/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
12202#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
12203
12204/** Prepares for using the AVX state.
12205 * Ensures that we can use the host AVX/FPU in the current context (RC+R0.
12206 * Ensures the guest AVX state in the CPUMCTX is up to date.
12207 * @note This will include the AVX512 state too when support for it is added
12208 * due to the zero extending feature of VEX instruction. */
12209#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
12210/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
12211#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
12212/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
12213#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
12214
12215/**
12216 * Calls a MMX assembly implementation taking two visible arguments.
12217 *
12218 * @param a_pfnAImpl Pointer to the assembly MMX routine.
12219 * @param a0 The first extra argument.
12220 * @param a1 The second extra argument.
12221 */
12222#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
12223 do { \
12224 IEM_MC_PREPARE_FPU_USAGE(); \
12225 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
12226 } while (0)
12227
12228/**
12229 * Calls a MMX assembly implementation taking three visible arguments.
12230 *
12231 * @param a_pfnAImpl Pointer to the assembly MMX routine.
12232 * @param a0 The first extra argument.
12233 * @param a1 The second extra argument.
12234 * @param a2 The third extra argument.
12235 */
12236#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12237 do { \
12238 IEM_MC_PREPARE_FPU_USAGE(); \
12239 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
12240 } while (0)
12241
12242
12243/**
12244 * Calls a SSE assembly implementation taking two visible arguments.
12245 *
12246 * @param a_pfnAImpl Pointer to the assembly SSE routine.
12247 * @param a0 The first extra argument.
12248 * @param a1 The second extra argument.
12249 */
12250#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
12251 do { \
12252 IEM_MC_PREPARE_SSE_USAGE(); \
12253 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1)); \
12254 } while (0)
12255
12256/**
12257 * Calls a SSE assembly implementation taking three visible arguments.
12258 *
12259 * @param a_pfnAImpl Pointer to the assembly SSE routine.
12260 * @param a0 The first extra argument.
12261 * @param a1 The second extra argument.
12262 * @param a2 The third extra argument.
12263 */
12264#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
12265 do { \
12266 IEM_MC_PREPARE_SSE_USAGE(); \
12267 a_pfnAImpl(&pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87, (a0), (a1), (a2)); \
12268 } while (0)
12269
12270
12271/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
12272 * IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ... */
12273#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
12274 IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, pVCpu->cpum.GstCtx.CTX_SUFF(pXState), 0)
12275
12276/**
12277 * Calls a AVX assembly implementation taking two visible arguments.
12278 *
12279 * There is one implicit zero'th argument, a pointer to the extended state.
12280 *
12281 * @param a_pfnAImpl Pointer to the assembly AVX routine.
12282 * @param a1 The first extra argument.
12283 * @param a2 The second extra argument.
12284 */
12285#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
12286 do { \
12287 IEM_MC_PREPARE_AVX_USAGE(); \
12288 a_pfnAImpl(pXState, (a1), (a2)); \
12289 } while (0)
12290
12291/**
12292 * Calls a AVX assembly implementation taking three visible arguments.
12293 *
12294 * There is one implicit zero'th argument, a pointer to the extended state.
12295 *
12296 * @param a_pfnAImpl Pointer to the assembly AVX routine.
12297 * @param a1 The first extra argument.
12298 * @param a2 The second extra argument.
12299 * @param a3 The third extra argument.
12300 */
12301#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
12302 do { \
12303 IEM_MC_PREPARE_AVX_USAGE(); \
12304 a_pfnAImpl(pXState, (a1), (a2), (a3)); \
12305 } while (0)
12306
12307/** @note Not for IOPL or IF testing. */
12308#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
12309/** @note Not for IOPL or IF testing. */
12310#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
12311/** @note Not for IOPL or IF testing. */
12312#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
12313/** @note Not for IOPL or IF testing. */
12314#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
12315/** @note Not for IOPL or IF testing. */
12316#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
12317 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12318 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12319/** @note Not for IOPL or IF testing. */
12320#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
12321 if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12322 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12323/** @note Not for IOPL or IF testing. */
12324#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
12325 if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
12326 || !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12327 != !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12328/** @note Not for IOPL or IF testing. */
12329#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
12330 if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
12331 && !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
12332 == !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
12333#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
12334#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
12335#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
12336/** @note Not for IOPL or IF testing. */
12337#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12338 if ( pVCpu->cpum.GstCtx.cx != 0 \
12339 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12340/** @note Not for IOPL or IF testing. */
12341#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12342 if ( pVCpu->cpum.GstCtx.ecx != 0 \
12343 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12344/** @note Not for IOPL or IF testing. */
12345#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
12346 if ( pVCpu->cpum.GstCtx.rcx != 0 \
12347 && (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12348/** @note Not for IOPL or IF testing. */
12349#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12350 if ( pVCpu->cpum.GstCtx.cx != 0 \
12351 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12352/** @note Not for IOPL or IF testing. */
12353#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12354 if ( pVCpu->cpum.GstCtx.ecx != 0 \
12355 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12356/** @note Not for IOPL or IF testing. */
12357#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
12358 if ( pVCpu->cpum.GstCtx.rcx != 0 \
12359 && !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
12360#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
12361#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
12362
12363#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
12364 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
12365#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
12366 if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
12367#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
12368 if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
12369#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
12370 if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
12371#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
12372 if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
12373#define IEM_MC_IF_FCW_IM() \
12374 if (pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87.FCW & X86_FCW_IM) {
12375
12376#define IEM_MC_ELSE() } else {
12377#define IEM_MC_ENDIF() } do {} while (0)
12378
12379/** @} */
12380
12381
12382/** @name Opcode Debug Helpers.
12383 * @{
12384 */
12385#ifdef VBOX_WITH_STATISTICS
12386# define IEMOP_INC_STATS(a_Stats) do { pVCpu->iem.s.CTX_SUFF(pStats)->a_Stats += 1; } while (0)
12387#else
12388# define IEMOP_INC_STATS(a_Stats) do { } while (0)
12389#endif
12390
12391#ifdef DEBUG
12392# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) \
12393 do { \
12394 IEMOP_INC_STATS(a_Stats); \
12395 Log4(("decode - %04x:%RGv %s%s [#%u]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, \
12396 pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK ? "lock " : "", a_szMnemonic, pVCpu->iem.s.cInstructions)); \
12397 } while (0)
12398
12399# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12400 do { \
12401 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12402 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12403 (void)RT_CONCAT(OP_,a_Upper); \
12404 (void)(a_fDisHints); \
12405 (void)(a_fIemHints); \
12406 } while (0)
12407
12408# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12409 do { \
12410 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12411 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12412 (void)RT_CONCAT(OP_,a_Upper); \
12413 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12414 (void)(a_fDisHints); \
12415 (void)(a_fIemHints); \
12416 } while (0)
12417
12418# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12419 do { \
12420 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12421 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12422 (void)RT_CONCAT(OP_,a_Upper); \
12423 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12424 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12425 (void)(a_fDisHints); \
12426 (void)(a_fIemHints); \
12427 } while (0)
12428
12429# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12430 do { \
12431 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12432 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12433 (void)RT_CONCAT(OP_,a_Upper); \
12434 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12435 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12436 (void)RT_CONCAT(OP_PARM_,a_Op3); \
12437 (void)(a_fDisHints); \
12438 (void)(a_fIemHints); \
12439 } while (0)
12440
12441# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12442 do { \
12443 IEMOP_MNEMONIC(a_Stats, a_szMnemonic); \
12444 (void)RT_CONCAT(IEMOPFORM_, a_Form); \
12445 (void)RT_CONCAT(OP_,a_Upper); \
12446 (void)RT_CONCAT(OP_PARM_,a_Op1); \
12447 (void)RT_CONCAT(OP_PARM_,a_Op2); \
12448 (void)RT_CONCAT(OP_PARM_,a_Op3); \
12449 (void)RT_CONCAT(OP_PARM_,a_Op4); \
12450 (void)(a_fDisHints); \
12451 (void)(a_fIemHints); \
12452 } while (0)
12453
12454#else
12455# define IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats)
12456
12457# define IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12458 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12459# define IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12460 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12461# define IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12462 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12463# define IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12464 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12465# define IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12466 IEMOP_MNEMONIC(a_Stats, a_szMnemonic)
12467
12468#endif
12469
12470#define IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints) \
12471 IEMOP_MNEMONIC0EX(a_Lower, \
12472 #a_Lower, \
12473 a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
12474#define IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints) \
12475 IEMOP_MNEMONIC1EX(RT_CONCAT3(a_Lower,_,a_Op1), \
12476 #a_Lower " " #a_Op1, \
12477 a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
12478#define IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints) \
12479 IEMOP_MNEMONIC2EX(RT_CONCAT5(a_Lower,_,a_Op1,_,a_Op2), \
12480 #a_Lower " " #a_Op1 "," #a_Op2, \
12481 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
12482#define IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints) \
12483 IEMOP_MNEMONIC3EX(RT_CONCAT7(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3), \
12484 #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3, \
12485 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
12486#define IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints) \
12487 IEMOP_MNEMONIC4EX(RT_CONCAT9(a_Lower,_,a_Op1,_,a_Op2,_,a_Op3,_,a_Op4), \
12488 #a_Lower " " #a_Op1 "," #a_Op2 "," #a_Op3 "," #a_Op4, \
12489 a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints)
12490
12491/** @} */
12492
12493
12494/** @name Opcode Helpers.
12495 * @{
12496 */
12497
12498#ifdef IN_RING3
12499# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
12500 do { \
12501 if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
12502 else \
12503 { \
12504 (void)DBGFSTOP(pVCpu->CTX_SUFF(pVM)); \
12505 return IEMOP_RAISE_INVALID_OPCODE(); \
12506 } \
12507 } while (0)
12508#else
12509# define IEMOP_HLP_MIN_CPU(a_uMinCpu, a_fOnlyIf) \
12510 do { \
12511 if (IEM_GET_TARGET_CPU(pVCpu) >= (a_uMinCpu) || !(a_fOnlyIf)) { } \
12512 else return IEMOP_RAISE_INVALID_OPCODE(); \
12513 } while (0)
12514#endif
12515
12516/** The instruction requires a 186 or later. */
12517#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_186
12518# define IEMOP_HLP_MIN_186() do { } while (0)
12519#else
12520# define IEMOP_HLP_MIN_186() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_186, true)
12521#endif
12522
12523/** The instruction requires a 286 or later. */
12524#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_286
12525# define IEMOP_HLP_MIN_286() do { } while (0)
12526#else
12527# define IEMOP_HLP_MIN_286() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_286, true)
12528#endif
12529
12530/** The instruction requires a 386 or later. */
12531#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
12532# define IEMOP_HLP_MIN_386() do { } while (0)
12533#else
12534# define IEMOP_HLP_MIN_386() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, true)
12535#endif
12536
12537/** The instruction requires a 386 or later if the given expression is true. */
12538#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_386
12539# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) do { } while (0)
12540#else
12541# define IEMOP_HLP_MIN_386_EX(a_fOnlyIf) IEMOP_HLP_MIN_CPU(IEMTARGETCPU_386, a_fOnlyIf)
12542#endif
12543
12544/** The instruction requires a 486 or later. */
12545#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_486
12546# define IEMOP_HLP_MIN_486() do { } while (0)
12547#else
12548# define IEMOP_HLP_MIN_486() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_486, true)
12549#endif
12550
12551/** The instruction requires a Pentium (586) or later. */
12552#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PENTIUM
12553# define IEMOP_HLP_MIN_586() do { } while (0)
12554#else
12555# define IEMOP_HLP_MIN_586() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PENTIUM, true)
12556#endif
12557
12558/** The instruction requires a PentiumPro (686) or later. */
12559#if IEM_CFG_TARGET_CPU >= IEMTARGETCPU_PPRO
12560# define IEMOP_HLP_MIN_686() do { } while (0)
12561#else
12562# define IEMOP_HLP_MIN_686() IEMOP_HLP_MIN_CPU(IEMTARGETCPU_PPRO, true)
12563#endif
12564
12565
12566/** The instruction raises an \#UD in real and V8086 mode. */
12567#define IEMOP_HLP_NO_REAL_OR_V86_MODE() \
12568 do \
12569 { \
12570 if (!IEM_IS_REAL_OR_V86_MODE(pVCpu)) { /* likely */ } \
12571 else return IEMOP_RAISE_INVALID_OPCODE(); \
12572 } while (0)
12573
12574#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12575/** This instruction raises an \#UD in real and V8086 mode or when not using a
12576 * 64-bit code segment when in long mode (applicable to all VMX instructions
12577 * except VMCALL). */
12578# define IEMOP_HLP_VMX_INSTR() \
12579 do \
12580 { \
12581 if ( !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
12582 && ( !IEM_IS_LONG_MODE(pVCpu) \
12583 || IEM_IS_64BIT_CODE(pVCpu))) \
12584 { /* likely */ } \
12585 else \
12586 return IEMOP_RAISE_INVALID_OPCODE(); \
12587 } while (0)
12588#endif
12589
12590/** The instruction is not available in 64-bit mode, throw \#UD if we're in
12591 * 64-bit mode. */
12592#define IEMOP_HLP_NO_64BIT() \
12593 do \
12594 { \
12595 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12596 return IEMOP_RAISE_INVALID_OPCODE(); \
12597 } while (0)
12598
12599/** The instruction is only available in 64-bit mode, throw \#UD if we're not in
12600 * 64-bit mode. */
12601#define IEMOP_HLP_ONLY_64BIT() \
12602 do \
12603 { \
12604 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT) \
12605 return IEMOP_RAISE_INVALID_OPCODE(); \
12606 } while (0)
12607
12608/** The instruction defaults to 64-bit operand size if 64-bit mode. */
12609#define IEMOP_HLP_DEFAULT_64BIT_OP_SIZE() \
12610 do \
12611 { \
12612 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12613 iemRecalEffOpSize64Default(pVCpu); \
12614 } while (0)
12615
12616/** The instruction has 64-bit operand size if 64-bit mode. */
12617#define IEMOP_HLP_64BIT_OP_SIZE() \
12618 do \
12619 { \
12620 if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT) \
12621 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT; \
12622 } while (0)
12623
12624/** Only a REX prefix immediately preceeding the first opcode byte takes
12625 * effect. This macro helps ensuring this as well as logging bad guest code. */
12626#define IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE(a_szPrf) \
12627 do \
12628 { \
12629 if (RT_UNLIKELY(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX)) \
12630 { \
12631 Log5((a_szPrf ": Overriding REX prefix at %RX16! fPrefixes=%#x\n", pVCpu->cpum.GstCtx.rip, pVCpu->iem.s.fPrefixes)); \
12632 pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REX_MASK; \
12633 pVCpu->iem.s.uRexB = 0; \
12634 pVCpu->iem.s.uRexIndex = 0; \
12635 pVCpu->iem.s.uRexReg = 0; \
12636 iemRecalEffOpSize(pVCpu); \
12637 } \
12638 } while (0)
12639
12640/**
12641 * Done decoding.
12642 */
12643#define IEMOP_HLP_DONE_DECODING() \
12644 do \
12645 { \
12646 /*nothing for now, maybe later... */ \
12647 } while (0)
12648
12649/**
12650 * Done decoding, raise \#UD exception if lock prefix present.
12651 */
12652#define IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX() \
12653 do \
12654 { \
12655 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12656 { /* likely */ } \
12657 else \
12658 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12659 } while (0)
12660
12661
12662/**
12663 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12664 * repnz or size prefixes are present, or if in real or v8086 mode.
12665 */
12666#define IEMOP_HLP_DONE_VEX_DECODING() \
12667 do \
12668 { \
12669 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12670 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12671 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
12672 { /* likely */ } \
12673 else \
12674 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12675 } while (0)
12676
12677/**
12678 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12679 * repnz or size prefixes are present, or if in real or v8086 mode.
12680 */
12681#define IEMOP_HLP_DONE_VEX_DECODING_L0() \
12682 do \
12683 { \
12684 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12685 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12686 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) \
12687 && pVCpu->iem.s.uVexLength == 0)) \
12688 { /* likely */ } \
12689 else \
12690 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12691 } while (0)
12692
12693
12694/**
12695 * Done decoding VEX instruction, raise \#UD exception if any lock, rex, repz,
12696 * repnz or size prefixes are present, or if the VEX.VVVV field doesn't indicate
12697 * register 0, or if in real or v8086 mode.
12698 */
12699#define IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV() \
12700 do \
12701 { \
12702 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12703 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REX)) \
12704 && !pVCpu->iem.s.uVex3rdReg \
12705 && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )) \
12706 { /* likely */ } \
12707 else \
12708 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12709 } while (0)
12710
12711/**
12712 * Done decoding VEX, no V, L=0.
12713 * Raises \#UD exception if rex, rep, opsize or lock prefixes are present, if
12714 * we're in real or v8086 mode, if VEX.V!=0xf, or if VEX.L!=0.
12715 */
12716#define IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV() \
12717 do \
12718 { \
12719 if (RT_LIKELY( !( pVCpu->iem.s.fPrefixes \
12720 & (IEM_OP_PRF_LOCK | IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REX)) \
12721 && pVCpu->iem.s.uVexLength == 0 \
12722 && pVCpu->iem.s.uVex3rdReg == 0 \
12723 && !IEM_IS_REAL_OR_V86_MODE(pVCpu))) \
12724 { /* likely */ } \
12725 else \
12726 return IEMOP_RAISE_INVALID_OPCODE(); \
12727 } while (0)
12728
12729#define IEMOP_HLP_DECODED_NL_1(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_fDisOpType) \
12730 do \
12731 { \
12732 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12733 { /* likely */ } \
12734 else \
12735 { \
12736 NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_fDisOpType); \
12737 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12738 } \
12739 } while (0)
12740#define IEMOP_HLP_DECODED_NL_2(a_uDisOpNo, a_fIemOpFlags, a_uDisParam0, a_uDisParam1, a_fDisOpType) \
12741 do \
12742 { \
12743 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))) \
12744 { /* likely */ } \
12745 else \
12746 { \
12747 NOREF(a_uDisOpNo); NOREF(a_fIemOpFlags); NOREF(a_uDisParam0); NOREF(a_uDisParam1); NOREF(a_fDisOpType); \
12748 return IEMOP_RAISE_INVALID_LOCK_PREFIX(); \
12749 } \
12750 } while (0)
12751
12752/**
12753 * Done decoding, raise \#UD exception if any lock, repz or repnz prefixes
12754 * are present.
12755 */
12756#define IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES() \
12757 do \
12758 { \
12759 if (RT_LIKELY(!(pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_LOCK | IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ)))) \
12760 { /* likely */ } \
12761 else \
12762 return IEMOP_RAISE_INVALID_OPCODE(); \
12763 } while (0)
12764
12765
12766/**
12767 * Calculates the effective address of a ModR/M memory operand.
12768 *
12769 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
12770 *
12771 * @return Strict VBox status code.
12772 * @param pVCpu The cross context virtual CPU structure of the calling thread.
12773 * @param bRm The ModRM byte.
12774 * @param cbImm The size of any immediate following the
12775 * effective address opcode bytes. Important for
12776 * RIP relative addressing.
12777 * @param pGCPtrEff Where to return the effective address.
12778 */
12779IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPU pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff)
12780{
12781 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
12782# define SET_SS_DEF() \
12783 do \
12784 { \
12785 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
12786 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
12787 } while (0)
12788
12789 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
12790 {
12791/** @todo Check the effective address size crap! */
12792 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
12793 {
12794 uint16_t u16EffAddr;
12795
12796 /* Handle the disp16 form with no registers first. */
12797 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
12798 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
12799 else
12800 {
12801 /* Get the displacment. */
12802 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
12803 {
12804 case 0: u16EffAddr = 0; break;
12805 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
12806 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
12807 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
12808 }
12809
12810 /* Add the base and index registers to the disp. */
12811 switch (bRm & X86_MODRM_RM_MASK)
12812 {
12813 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
12814 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
12815 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
12816 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
12817 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
12818 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
12819 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
12820 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
12821 }
12822 }
12823
12824 *pGCPtrEff = u16EffAddr;
12825 }
12826 else
12827 {
12828 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
12829 uint32_t u32EffAddr;
12830
12831 /* Handle the disp32 form with no registers first. */
12832 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
12833 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
12834 else
12835 {
12836 /* Get the register (or SIB) value. */
12837 switch ((bRm & X86_MODRM_RM_MASK))
12838 {
12839 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
12840 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
12841 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
12842 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
12843 case 4: /* SIB */
12844 {
12845 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
12846
12847 /* Get the index and scale it. */
12848 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
12849 {
12850 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
12851 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
12852 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
12853 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
12854 case 4: u32EffAddr = 0; /*none */ break;
12855 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
12856 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
12857 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
12858 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12859 }
12860 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
12861
12862 /* add base */
12863 switch (bSib & X86_SIB_BASE_MASK)
12864 {
12865 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
12866 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
12867 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
12868 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
12869 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
12870 case 5:
12871 if ((bRm & X86_MODRM_MOD_MASK) != 0)
12872 {
12873 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
12874 SET_SS_DEF();
12875 }
12876 else
12877 {
12878 uint32_t u32Disp;
12879 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
12880 u32EffAddr += u32Disp;
12881 }
12882 break;
12883 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
12884 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
12885 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12886 }
12887 break;
12888 }
12889 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
12890 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
12891 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
12892 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12893 }
12894
12895 /* Get and add the displacement. */
12896 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
12897 {
12898 case 0:
12899 break;
12900 case 1:
12901 {
12902 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
12903 u32EffAddr += i8Disp;
12904 break;
12905 }
12906 case 2:
12907 {
12908 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
12909 u32EffAddr += u32Disp;
12910 break;
12911 }
12912 default:
12913 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
12914 }
12915
12916 }
12917 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
12918 *pGCPtrEff = u32EffAddr;
12919 else
12920 {
12921 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
12922 *pGCPtrEff = u32EffAddr & UINT16_MAX;
12923 }
12924 }
12925 }
12926 else
12927 {
12928 uint64_t u64EffAddr;
12929
12930 /* Handle the rip+disp32 form with no registers first. */
12931 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
12932 {
12933 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
12934 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
12935 }
12936 else
12937 {
12938 /* Get the register (or SIB) value. */
12939 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
12940 {
12941 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
12942 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
12943 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
12944 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
12945 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
12946 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
12947 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
12948 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
12949 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
12950 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
12951 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
12952 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
12953 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
12954 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
12955 /* SIB */
12956 case 4:
12957 case 12:
12958 {
12959 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
12960
12961 /* Get the index and scale it. */
12962 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
12963 {
12964 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
12965 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
12966 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
12967 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
12968 case 4: u64EffAddr = 0; /*none */ break;
12969 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
12970 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
12971 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
12972 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
12973 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
12974 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
12975 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
12976 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
12977 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
12978 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
12979 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
12980 IEM_NOT_REACHED_DEFAULT_CASE_RET();
12981 }
12982 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
12983
12984 /* add base */
12985 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
12986 {
12987 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
12988 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
12989 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
12990 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
12991 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
12992 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
12993 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
12994 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
12995 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
12996 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
12997 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
12998 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
12999 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13000 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13001 /* complicated encodings */
13002 case 5:
13003 case 13:
13004 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13005 {
13006 if (!pVCpu->iem.s.uRexB)
13007 {
13008 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13009 SET_SS_DEF();
13010 }
13011 else
13012 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13013 }
13014 else
13015 {
13016 uint32_t u32Disp;
13017 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13018 u64EffAddr += (int32_t)u32Disp;
13019 }
13020 break;
13021 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13022 }
13023 break;
13024 }
13025 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13026 }
13027
13028 /* Get and add the displacement. */
13029 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13030 {
13031 case 0:
13032 break;
13033 case 1:
13034 {
13035 int8_t i8Disp;
13036 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13037 u64EffAddr += i8Disp;
13038 break;
13039 }
13040 case 2:
13041 {
13042 uint32_t u32Disp;
13043 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13044 u64EffAddr += (int32_t)u32Disp;
13045 break;
13046 }
13047 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
13048 }
13049
13050 }
13051
13052 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13053 *pGCPtrEff = u64EffAddr;
13054 else
13055 {
13056 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13057 *pGCPtrEff = u64EffAddr & UINT32_MAX;
13058 }
13059 }
13060
13061 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
13062 return VINF_SUCCESS;
13063}
13064
13065
13066/**
13067 * Calculates the effective address of a ModR/M memory operand.
13068 *
13069 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
13070 *
13071 * @return Strict VBox status code.
13072 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13073 * @param bRm The ModRM byte.
13074 * @param cbImm The size of any immediate following the
13075 * effective address opcode bytes. Important for
13076 * RIP relative addressing.
13077 * @param pGCPtrEff Where to return the effective address.
13078 * @param offRsp RSP displacement.
13079 */
13080IEM_STATIC VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPU pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp)
13081{
13082 Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
13083# define SET_SS_DEF() \
13084 do \
13085 { \
13086 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
13087 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
13088 } while (0)
13089
13090 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
13091 {
13092/** @todo Check the effective address size crap! */
13093 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
13094 {
13095 uint16_t u16EffAddr;
13096
13097 /* Handle the disp16 form with no registers first. */
13098 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
13099 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
13100 else
13101 {
13102 /* Get the displacment. */
13103 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13104 {
13105 case 0: u16EffAddr = 0; break;
13106 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
13107 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
13108 default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
13109 }
13110
13111 /* Add the base and index registers to the disp. */
13112 switch (bRm & X86_MODRM_RM_MASK)
13113 {
13114 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
13115 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
13116 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
13117 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
13118 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
13119 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
13120 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
13121 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
13122 }
13123 }
13124
13125 *pGCPtrEff = u16EffAddr;
13126 }
13127 else
13128 {
13129 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13130 uint32_t u32EffAddr;
13131
13132 /* Handle the disp32 form with no registers first. */
13133 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13134 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
13135 else
13136 {
13137 /* Get the register (or SIB) value. */
13138 switch ((bRm & X86_MODRM_RM_MASK))
13139 {
13140 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13141 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13142 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13143 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13144 case 4: /* SIB */
13145 {
13146 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13147
13148 /* Get the index and scale it. */
13149 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
13150 {
13151 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13152 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13153 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13154 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13155 case 4: u32EffAddr = 0; /*none */ break;
13156 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
13157 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13158 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13159 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13160 }
13161 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13162
13163 /* add base */
13164 switch (bSib & X86_SIB_BASE_MASK)
13165 {
13166 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
13167 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
13168 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
13169 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
13170 case 4:
13171 u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
13172 SET_SS_DEF();
13173 break;
13174 case 5:
13175 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13176 {
13177 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
13178 SET_SS_DEF();
13179 }
13180 else
13181 {
13182 uint32_t u32Disp;
13183 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13184 u32EffAddr += u32Disp;
13185 }
13186 break;
13187 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13188 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13189 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13190 }
13191 break;
13192 }
13193 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13194 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13195 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13196 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13197 }
13198
13199 /* Get and add the displacement. */
13200 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13201 {
13202 case 0:
13203 break;
13204 case 1:
13205 {
13206 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13207 u32EffAddr += i8Disp;
13208 break;
13209 }
13210 case 2:
13211 {
13212 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13213 u32EffAddr += u32Disp;
13214 break;
13215 }
13216 default:
13217 AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
13218 }
13219
13220 }
13221 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13222 *pGCPtrEff = u32EffAddr;
13223 else
13224 {
13225 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13226 *pGCPtrEff = u32EffAddr & UINT16_MAX;
13227 }
13228 }
13229 }
13230 else
13231 {
13232 uint64_t u64EffAddr;
13233
13234 /* Handle the rip+disp32 form with no registers first. */
13235 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13236 {
13237 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13238 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13239 }
13240 else
13241 {
13242 /* Get the register (or SIB) value. */
13243 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13244 {
13245 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13246 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13247 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13248 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13249 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13250 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13251 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13252 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13253 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13254 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13255 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13256 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13257 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13258 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13259 /* SIB */
13260 case 4:
13261 case 12:
13262 {
13263 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13264
13265 /* Get the index and scale it. */
13266 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13267 {
13268 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13269 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13270 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13271 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13272 case 4: u64EffAddr = 0; /*none */ break;
13273 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13274 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13275 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13276 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13277 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13278 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13279 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13280 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13281 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13282 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13283 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13284 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13285 }
13286 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13287
13288 /* add base */
13289 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13290 {
13291 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13292 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13293 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13294 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13295 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
13296 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13297 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13298 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13299 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13300 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13301 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13302 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13303 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13304 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13305 /* complicated encodings */
13306 case 5:
13307 case 13:
13308 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13309 {
13310 if (!pVCpu->iem.s.uRexB)
13311 {
13312 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13313 SET_SS_DEF();
13314 }
13315 else
13316 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13317 }
13318 else
13319 {
13320 uint32_t u32Disp;
13321 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13322 u64EffAddr += (int32_t)u32Disp;
13323 }
13324 break;
13325 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13326 }
13327 break;
13328 }
13329 IEM_NOT_REACHED_DEFAULT_CASE_RET();
13330 }
13331
13332 /* Get and add the displacement. */
13333 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13334 {
13335 case 0:
13336 break;
13337 case 1:
13338 {
13339 int8_t i8Disp;
13340 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13341 u64EffAddr += i8Disp;
13342 break;
13343 }
13344 case 2:
13345 {
13346 uint32_t u32Disp;
13347 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13348 u64EffAddr += (int32_t)u32Disp;
13349 break;
13350 }
13351 IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
13352 }
13353
13354 }
13355
13356 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13357 *pGCPtrEff = u64EffAddr;
13358 else
13359 {
13360 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13361 *pGCPtrEff = u64EffAddr & UINT32_MAX;
13362 }
13363 }
13364
13365 Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
13366 return VINF_SUCCESS;
13367}
13368
13369
13370#ifdef IEM_WITH_SETJMP
13371/**
13372 * Calculates the effective address of a ModR/M memory operand.
13373 *
13374 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
13375 *
13376 * May longjmp on internal error.
13377 *
13378 * @return The effective address.
13379 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13380 * @param bRm The ModRM byte.
13381 * @param cbImm The size of any immediate following the
13382 * effective address opcode bytes. Important for
13383 * RIP relative addressing.
13384 */
13385IEM_STATIC RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPU pVCpu, uint8_t bRm, uint8_t cbImm)
13386{
13387 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
13388# define SET_SS_DEF() \
13389 do \
13390 { \
13391 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
13392 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
13393 } while (0)
13394
13395 if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
13396 {
13397/** @todo Check the effective address size crap! */
13398 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
13399 {
13400 uint16_t u16EffAddr;
13401
13402 /* Handle the disp16 form with no registers first. */
13403 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
13404 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
13405 else
13406 {
13407 /* Get the displacment. */
13408 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13409 {
13410 case 0: u16EffAddr = 0; break;
13411 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
13412 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
13413 default: AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_1)); /* (caller checked for these) */
13414 }
13415
13416 /* Add the base and index registers to the disp. */
13417 switch (bRm & X86_MODRM_RM_MASK)
13418 {
13419 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
13420 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
13421 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
13422 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
13423 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
13424 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
13425 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
13426 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
13427 }
13428 }
13429
13430 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
13431 return u16EffAddr;
13432 }
13433
13434 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13435 uint32_t u32EffAddr;
13436
13437 /* Handle the disp32 form with no registers first. */
13438 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13439 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
13440 else
13441 {
13442 /* Get the register (or SIB) value. */
13443 switch ((bRm & X86_MODRM_RM_MASK))
13444 {
13445 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13446 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13447 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13448 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13449 case 4: /* SIB */
13450 {
13451 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13452
13453 /* Get the index and scale it. */
13454 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
13455 {
13456 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
13457 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
13458 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
13459 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
13460 case 4: u32EffAddr = 0; /*none */ break;
13461 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
13462 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13463 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13464 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13465 }
13466 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13467
13468 /* add base */
13469 switch (bSib & X86_SIB_BASE_MASK)
13470 {
13471 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
13472 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
13473 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
13474 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
13475 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
13476 case 5:
13477 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13478 {
13479 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
13480 SET_SS_DEF();
13481 }
13482 else
13483 {
13484 uint32_t u32Disp;
13485 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13486 u32EffAddr += u32Disp;
13487 }
13488 break;
13489 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
13490 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
13491 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13492 }
13493 break;
13494 }
13495 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
13496 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
13497 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
13498 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13499 }
13500
13501 /* Get and add the displacement. */
13502 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13503 {
13504 case 0:
13505 break;
13506 case 1:
13507 {
13508 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13509 u32EffAddr += i8Disp;
13510 break;
13511 }
13512 case 2:
13513 {
13514 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13515 u32EffAddr += u32Disp;
13516 break;
13517 }
13518 default:
13519 AssertFailedStmt(longjmp(*pVCpu->iem.s.CTX_SUFF(pJmpBuf), VERR_IEM_IPE_2)); /* (caller checked for these) */
13520 }
13521 }
13522
13523 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
13524 {
13525 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
13526 return u32EffAddr;
13527 }
13528 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
13529 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
13530 return u32EffAddr & UINT16_MAX;
13531 }
13532
13533 uint64_t u64EffAddr;
13534
13535 /* Handle the rip+disp32 form with no registers first. */
13536 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
13537 {
13538 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
13539 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
13540 }
13541 else
13542 {
13543 /* Get the register (or SIB) value. */
13544 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
13545 {
13546 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13547 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13548 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13549 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13550 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
13551 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13552 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13553 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13554 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13555 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13556 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13557 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13558 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13559 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13560 /* SIB */
13561 case 4:
13562 case 12:
13563 {
13564 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
13565
13566 /* Get the index and scale it. */
13567 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
13568 {
13569 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
13570 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
13571 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
13572 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
13573 case 4: u64EffAddr = 0; /*none */ break;
13574 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
13575 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
13576 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
13577 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
13578 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
13579 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
13580 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
13581 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
13582 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
13583 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
13584 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
13585 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13586 }
13587 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
13588
13589 /* add base */
13590 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
13591 {
13592 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
13593 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
13594 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
13595 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
13596 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
13597 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
13598 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
13599 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
13600 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
13601 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
13602 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
13603 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
13604 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
13605 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
13606 /* complicated encodings */
13607 case 5:
13608 case 13:
13609 if ((bRm & X86_MODRM_MOD_MASK) != 0)
13610 {
13611 if (!pVCpu->iem.s.uRexB)
13612 {
13613 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
13614 SET_SS_DEF();
13615 }
13616 else
13617 u64EffAddr += pVCpu->cpum.GstCtx.r13;
13618 }
13619 else
13620 {
13621 uint32_t u32Disp;
13622 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13623 u64EffAddr += (int32_t)u32Disp;
13624 }
13625 break;
13626 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13627 }
13628 break;
13629 }
13630 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
13631 }
13632
13633 /* Get and add the displacement. */
13634 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
13635 {
13636 case 0:
13637 break;
13638 case 1:
13639 {
13640 int8_t i8Disp;
13641 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
13642 u64EffAddr += i8Disp;
13643 break;
13644 }
13645 case 2:
13646 {
13647 uint32_t u32Disp;
13648 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
13649 u64EffAddr += (int32_t)u32Disp;
13650 break;
13651 }
13652 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
13653 }
13654
13655 }
13656
13657 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
13658 {
13659 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
13660 return u64EffAddr;
13661 }
13662 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
13663 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
13664 return u64EffAddr & UINT32_MAX;
13665}
13666#endif /* IEM_WITH_SETJMP */
13667
13668/** @} */
13669
13670
13671
13672/*
13673 * Include the instructions
13674 */
13675#include "IEMAllInstructions.cpp.h"
13676
13677
13678
13679#ifdef LOG_ENABLED
13680/**
13681 * Logs the current instruction.
13682 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
13683 * @param fSameCtx Set if we have the same context information as the VMM,
13684 * clear if we may have already executed an instruction in
13685 * our debug context. When clear, we assume IEMCPU holds
13686 * valid CPU mode info.
13687 *
13688 * The @a fSameCtx parameter is now misleading and obsolete.
13689 * @param pszFunction The IEM function doing the execution.
13690 */
13691IEM_STATIC void iemLogCurInstr(PVMCPU pVCpu, bool fSameCtx, const char *pszFunction)
13692{
13693# ifdef IN_RING3
13694 if (LogIs2Enabled())
13695 {
13696 char szInstr[256];
13697 uint32_t cbInstr = 0;
13698 if (fSameCtx)
13699 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
13700 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
13701 szInstr, sizeof(szInstr), &cbInstr);
13702 else
13703 {
13704 uint32_t fFlags = 0;
13705 switch (pVCpu->iem.s.enmCpuMode)
13706 {
13707 case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;
13708 case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;
13709 case IEMMODE_16BIT:
13710 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
13711 fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
13712 else
13713 fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;
13714 break;
13715 }
13716 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
13717 szInstr, sizeof(szInstr), &cbInstr);
13718 }
13719
13720 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.CTX_SUFF(pXState)->x87;
13721 Log2(("**** %s\n"
13722 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
13723 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
13724 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
13725 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
13726 " %s\n"
13727 , pszFunction,
13728 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
13729 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
13730 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
13731 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
13732 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
13733 szInstr));
13734
13735 if (LogIs3Enabled())
13736 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
13737 }
13738 else
13739# endif
13740 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
13741 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
13742 RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
13743}
13744#endif /* LOG_ENABLED */
13745
13746
13747/**
13748 * Makes status code addjustments (pass up from I/O and access handler)
13749 * as well as maintaining statistics.
13750 *
13751 * @returns Strict VBox status code to pass up.
13752 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13753 * @param rcStrict The status from executing an instruction.
13754 */
13755DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
13756{
13757 if (rcStrict != VINF_SUCCESS)
13758 {
13759 if (RT_SUCCESS(rcStrict))
13760 {
13761 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
13762 || rcStrict == VINF_IOM_R3_IOPORT_READ
13763 || rcStrict == VINF_IOM_R3_IOPORT_WRITE
13764 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
13765 || rcStrict == VINF_IOM_R3_MMIO_READ
13766 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
13767 || rcStrict == VINF_IOM_R3_MMIO_WRITE
13768 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
13769 || rcStrict == VINF_CPUM_R3_MSR_READ
13770 || rcStrict == VINF_CPUM_R3_MSR_WRITE
13771 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
13772 || rcStrict == VINF_EM_RAW_TO_R3
13773 || rcStrict == VINF_EM_TRIPLE_FAULT
13774 || rcStrict == VINF_GIM_R3_HYPERCALL
13775 /* raw-mode / virt handlers only: */
13776 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
13777 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
13778 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
13779 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
13780 || rcStrict == VINF_SELM_SYNC_GDT
13781 || rcStrict == VINF_CSAM_PENDING_ACTION
13782 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
13783 /* nested hw.virt codes: */
13784 || rcStrict == VINF_SVM_VMEXIT
13785 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13786/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR */
13787 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
13788#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
13789 if ( rcStrict == VINF_SVM_VMEXIT
13790 && rcPassUp == VINF_SUCCESS)
13791 rcStrict = VINF_SUCCESS;
13792 else
13793#endif
13794 if (rcPassUp == VINF_SUCCESS)
13795 pVCpu->iem.s.cRetInfStatuses++;
13796 else if ( rcPassUp < VINF_EM_FIRST
13797 || rcPassUp > VINF_EM_LAST
13798 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
13799 {
13800 Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
13801 pVCpu->iem.s.cRetPassUpStatus++;
13802 rcStrict = rcPassUp;
13803 }
13804 else
13805 {
13806 Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
13807 pVCpu->iem.s.cRetInfStatuses++;
13808 }
13809 }
13810 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
13811 pVCpu->iem.s.cRetAspectNotImplemented++;
13812 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
13813 pVCpu->iem.s.cRetInstrNotImplemented++;
13814 else
13815 pVCpu->iem.s.cRetErrStatuses++;
13816 }
13817 else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
13818 {
13819 pVCpu->iem.s.cRetPassUpStatus++;
13820 rcStrict = pVCpu->iem.s.rcPassUp;
13821 }
13822
13823 return rcStrict;
13824}
13825
13826
13827/**
13828 * The actual code execution bits of IEMExecOne, IEMExecOneEx, and
13829 * IEMExecOneWithPrefetchedByPC.
13830 *
13831 * Similar code is found in IEMExecLots.
13832 *
13833 * @return Strict VBox status code.
13834 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
13835 * @param fExecuteInhibit If set, execute the instruction following CLI,
13836 * POP SS and MOV SS,GR.
13837 * @param pszFunction The calling function name.
13838 */
13839DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPU pVCpu, bool fExecuteInhibit, const char *pszFunction)
13840{
13841 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
13842 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
13843 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
13844 RT_NOREF_PV(pszFunction);
13845
13846#ifdef IEM_WITH_SETJMP
13847 VBOXSTRICTRC rcStrict;
13848 jmp_buf JmpBuf;
13849 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
13850 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
13851 if ((rcStrict = setjmp(JmpBuf)) == 0)
13852 {
13853 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
13854 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
13855 }
13856 else
13857 pVCpu->iem.s.cLongJumps++;
13858 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
13859#else
13860 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
13861 VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
13862#endif
13863 if (rcStrict == VINF_SUCCESS)
13864 pVCpu->iem.s.cInstructions++;
13865 if (pVCpu->iem.s.cActiveMappings > 0)
13866 {
13867 Assert(rcStrict != VINF_SUCCESS);
13868 iemMemRollback(pVCpu);
13869 }
13870 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
13871 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
13872 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
13873
13874//#ifdef DEBUG
13875// AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
13876//#endif
13877
13878 /* Execute the next instruction as well if a cli, pop ss or
13879 mov ss, Gr has just completed successfully. */
13880 if ( fExecuteInhibit
13881 && rcStrict == VINF_SUCCESS
13882 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
13883 && EMGetInhibitInterruptsPC(pVCpu) == pVCpu->cpum.GstCtx.rip )
13884 {
13885 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers);
13886 if (rcStrict == VINF_SUCCESS)
13887 {
13888#ifdef LOG_ENABLED
13889 iemLogCurInstr(pVCpu, false, pszFunction);
13890#endif
13891#ifdef IEM_WITH_SETJMP
13892 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
13893 if ((rcStrict = setjmp(JmpBuf)) == 0)
13894 {
13895 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
13896 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
13897 }
13898 else
13899 pVCpu->iem.s.cLongJumps++;
13900 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
13901#else
13902 IEM_OPCODE_GET_NEXT_U8(&b);
13903 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
13904#endif
13905 if (rcStrict == VINF_SUCCESS)
13906 pVCpu->iem.s.cInstructions++;
13907 if (pVCpu->iem.s.cActiveMappings > 0)
13908 {
13909 Assert(rcStrict != VINF_SUCCESS);
13910 iemMemRollback(pVCpu);
13911 }
13912 AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
13913 AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
13914 AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
13915 }
13916 else if (pVCpu->iem.s.cActiveMappings > 0)
13917 iemMemRollback(pVCpu);
13918 EMSetInhibitInterruptsPC(pVCpu, UINT64_C(0x7777555533331111));
13919 }
13920
13921 /*
13922 * Return value fiddling, statistics and sanity assertions.
13923 */
13924 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
13925
13926 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
13927 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
13928 return rcStrict;
13929}
13930
13931
13932#ifdef IN_RC
13933/**
13934 * Re-enters raw-mode or ensure we return to ring-3.
13935 *
13936 * @returns rcStrict, maybe modified.
13937 * @param pVCpu The cross context virtual CPU structure of the calling thread.
13938 * @param rcStrict The status code returne by the interpreter.
13939 */
13940DECLINLINE(VBOXSTRICTRC) iemRCRawMaybeReenter(PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
13941{
13942 if ( !pVCpu->iem.s.fInPatchCode
13943 && ( rcStrict == VINF_SUCCESS
13944 || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED /* pgmPoolAccessPfHandlerFlush */
13945 || rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED /* ditto */ ) )
13946 {
13947 if (pVCpu->cpum.GstCtx.eflags.Bits.u1IF || rcStrict != VINF_SUCCESS)
13948 CPUMRawEnter(pVCpu);
13949 else
13950 {
13951 Log(("iemRCRawMaybeReenter: VINF_EM_RESCHEDULE\n"));
13952 rcStrict = VINF_EM_RESCHEDULE;
13953 }
13954 }
13955 return rcStrict;
13956}
13957#endif
13958
13959
13960/**
13961 * Execute one instruction.
13962 *
13963 * @return Strict VBox status code.
13964 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
13965 */
13966VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPU pVCpu)
13967{
13968#ifdef LOG_ENABLED
13969 iemLogCurInstr(pVCpu, true, "IEMExecOne");
13970#endif
13971
13972 /*
13973 * Do the decoding and emulation.
13974 */
13975 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
13976 if (rcStrict == VINF_SUCCESS)
13977 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
13978 else if (pVCpu->iem.s.cActiveMappings > 0)
13979 iemMemRollback(pVCpu);
13980
13981#ifdef IN_RC
13982 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
13983#endif
13984 if (rcStrict != VINF_SUCCESS)
13985 LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
13986 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
13987 return rcStrict;
13988}
13989
13990
13991VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
13992{
13993 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
13994
13995 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
13996 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
13997 if (rcStrict == VINF_SUCCESS)
13998 {
13999 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
14000 if (pcbWritten)
14001 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14002 }
14003 else if (pVCpu->iem.s.cActiveMappings > 0)
14004 iemMemRollback(pVCpu);
14005
14006#ifdef IN_RC
14007 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14008#endif
14009 return rcStrict;
14010}
14011
14012
14013VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14014 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
14015{
14016 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14017
14018 VBOXSTRICTRC rcStrict;
14019 if ( cbOpcodeBytes
14020 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14021 {
14022 iemInitDecoder(pVCpu, false);
14023#ifdef IEM_WITH_CODE_TLB
14024 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14025 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14026 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14027 pVCpu->iem.s.offCurInstrStart = 0;
14028 pVCpu->iem.s.offInstrNextByte = 0;
14029#else
14030 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14031 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14032#endif
14033 rcStrict = VINF_SUCCESS;
14034 }
14035 else
14036 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14037 if (rcStrict == VINF_SUCCESS)
14038 rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
14039 else if (pVCpu->iem.s.cActiveMappings > 0)
14040 iemMemRollback(pVCpu);
14041
14042#ifdef IN_RC
14043 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14044#endif
14045 return rcStrict;
14046}
14047
14048
14049VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint32_t *pcbWritten)
14050{
14051 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14052
14053 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14054 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
14055 if (rcStrict == VINF_SUCCESS)
14056 {
14057 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
14058 if (pcbWritten)
14059 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14060 }
14061 else if (pVCpu->iem.s.cActiveMappings > 0)
14062 iemMemRollback(pVCpu);
14063
14064#ifdef IN_RC
14065 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14066#endif
14067 return rcStrict;
14068}
14069
14070
14071VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14072 const void *pvOpcodeBytes, size_t cbOpcodeBytes)
14073{
14074 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14075
14076 VBOXSTRICTRC rcStrict;
14077 if ( cbOpcodeBytes
14078 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14079 {
14080 iemInitDecoder(pVCpu, true);
14081#ifdef IEM_WITH_CODE_TLB
14082 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14083 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14084 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14085 pVCpu->iem.s.offCurInstrStart = 0;
14086 pVCpu->iem.s.offInstrNextByte = 0;
14087#else
14088 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14089 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14090#endif
14091 rcStrict = VINF_SUCCESS;
14092 }
14093 else
14094 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
14095 if (rcStrict == VINF_SUCCESS)
14096 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
14097 else if (pVCpu->iem.s.cActiveMappings > 0)
14098 iemMemRollback(pVCpu);
14099
14100#ifdef IN_RC
14101 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14102#endif
14103 return rcStrict;
14104}
14105
14106
14107/**
14108 * For debugging DISGetParamSize, may come in handy.
14109 *
14110 * @returns Strict VBox status code.
14111 * @param pVCpu The cross context virtual CPU structure of the
14112 * calling EMT.
14113 * @param pCtxCore The context core structure.
14114 * @param OpcodeBytesPC The PC of the opcode bytes.
14115 * @param pvOpcodeBytes Prefeched opcode bytes.
14116 * @param cbOpcodeBytes Number of prefetched bytes.
14117 * @param pcbWritten Where to return the number of bytes written.
14118 * Optional.
14119 */
14120VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPCWritten(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore, uint64_t OpcodeBytesPC,
14121 const void *pvOpcodeBytes, size_t cbOpcodeBytes,
14122 uint32_t *pcbWritten)
14123{
14124 AssertReturn(CPUMCTX2CORE(IEM_GET_CTX(pVCpu)) == pCtxCore, VERR_IEM_IPE_3);
14125
14126 uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
14127 VBOXSTRICTRC rcStrict;
14128 if ( cbOpcodeBytes
14129 && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
14130 {
14131 iemInitDecoder(pVCpu, true);
14132#ifdef IEM_WITH_CODE_TLB
14133 pVCpu->iem.s.uInstrBufPc = OpcodeBytesPC;
14134 pVCpu->iem.s.pbInstrBuf = (uint8_t const *)pvOpcodeBytes;
14135 pVCpu->iem.s.cbInstrBufTotal = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
14136 pVCpu->iem.s.offCurInstrStart = 0;
14137 pVCpu->iem.s.offInstrNextByte = 0;
14138#else
14139 pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
14140 memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
14141#endif
14142 rcStrict = VINF_SUCCESS;
14143 }
14144 else
14145 rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true);
14146 if (rcStrict == VINF_SUCCESS)
14147 {
14148 rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPCWritten");
14149 if (pcbWritten)
14150 *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
14151 }
14152 else if (pVCpu->iem.s.cActiveMappings > 0)
14153 iemMemRollback(pVCpu);
14154
14155#ifdef IN_RC
14156 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14157#endif
14158 return rcStrict;
14159}
14160
14161
14162VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPU pVCpu, uint32_t *pcInstructions)
14163{
14164 uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
14165
14166 /*
14167 * See if there is an interrupt pending in TRPM, inject it if we can.
14168 */
14169 /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
14170#if defined(VBOX_WITH_NESTED_HWVIRT_SVM)
14171 bool fIntrEnabled = pVCpu->cpum.GstCtx.hwvirt.fGif;
14172 if (fIntrEnabled)
14173 {
14174 if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
14175 fIntrEnabled = CPUMCanSvmNstGstTakePhysIntr(pVCpu, IEM_GET_CTX(pVCpu));
14176 else
14177 fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
14178 }
14179#else
14180 bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
14181#endif
14182 if ( fIntrEnabled
14183 && TRPMHasTrap(pVCpu)
14184 && EMGetInhibitInterruptsPC(pVCpu) != pVCpu->cpum.GstCtx.rip)
14185 {
14186 uint8_t u8TrapNo;
14187 TRPMEVENT enmType;
14188 RTGCUINT uErrCode;
14189 RTGCPTR uCr2;
14190 int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /* pu8InstLen */); AssertRC(rc2);
14191 IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /* cbInstr */);
14192 TRPMResetTrap(pVCpu);
14193 }
14194
14195 /*
14196 * Initial decoder init w/ prefetch, then setup setjmp.
14197 */
14198 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14199 if (rcStrict == VINF_SUCCESS)
14200 {
14201#ifdef IEM_WITH_SETJMP
14202 jmp_buf JmpBuf;
14203 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14204 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14205 pVCpu->iem.s.cActiveMappings = 0;
14206 if ((rcStrict = setjmp(JmpBuf)) == 0)
14207#endif
14208 {
14209 /*
14210 * The run loop. We limit ourselves to 4096 instructions right now.
14211 */
14212 PVM pVM = pVCpu->CTX_SUFF(pVM);
14213 uint32_t cInstr = 4096;
14214 for (;;)
14215 {
14216 /*
14217 * Log the state.
14218 */
14219#ifdef LOG_ENABLED
14220 iemLogCurInstr(pVCpu, true, "IEMExecLots");
14221#endif
14222
14223 /*
14224 * Do the decoding and emulation.
14225 */
14226 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14227 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14228 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14229 {
14230 Assert(pVCpu->iem.s.cActiveMappings == 0);
14231 pVCpu->iem.s.cInstructions++;
14232 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
14233 {
14234 uint32_t fCpu = pVCpu->fLocalForcedActions
14235 & ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
14236 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
14237 | VMCPU_FF_TLB_FLUSH
14238#ifdef VBOX_WITH_RAW_MODE
14239 | VMCPU_FF_TRPM_SYNC_IDT
14240 | VMCPU_FF_SELM_SYNC_TSS
14241 | VMCPU_FF_SELM_SYNC_GDT
14242 | VMCPU_FF_SELM_SYNC_LDT
14243#endif
14244 | VMCPU_FF_INHIBIT_INTERRUPTS
14245 | VMCPU_FF_BLOCK_NMIS
14246 | VMCPU_FF_UNHALT ));
14247
14248 if (RT_LIKELY( ( !fCpu
14249 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
14250 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
14251 && !VM_FF_IS_PENDING(pVM, VM_FF_ALL_MASK) ))
14252 {
14253 if (cInstr-- > 0)
14254 {
14255 Assert(pVCpu->iem.s.cActiveMappings == 0);
14256 iemReInitDecoder(pVCpu);
14257 continue;
14258 }
14259 }
14260 }
14261 Assert(pVCpu->iem.s.cActiveMappings == 0);
14262 }
14263 else if (pVCpu->iem.s.cActiveMappings > 0)
14264 iemMemRollback(pVCpu);
14265 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14266 break;
14267 }
14268 }
14269#ifdef IEM_WITH_SETJMP
14270 else
14271 {
14272 if (pVCpu->iem.s.cActiveMappings > 0)
14273 iemMemRollback(pVCpu);
14274 pVCpu->iem.s.cLongJumps++;
14275 }
14276 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14277#endif
14278
14279 /*
14280 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
14281 */
14282 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14283 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14284 }
14285 else
14286 {
14287 if (pVCpu->iem.s.cActiveMappings > 0)
14288 iemMemRollback(pVCpu);
14289
14290#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
14291 /*
14292 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
14293 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
14294 */
14295 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14296#endif
14297 }
14298
14299 /*
14300 * Maybe re-enter raw-mode and log.
14301 */
14302#ifdef IN_RC
14303 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14304#endif
14305 if (rcStrict != VINF_SUCCESS)
14306 LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14307 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14308 if (pcInstructions)
14309 *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
14310 return rcStrict;
14311}
14312
14313
14314/**
14315 * Interface used by EMExecuteExec, does exit statistics and limits.
14316 *
14317 * @returns Strict VBox status code.
14318 * @param pVCpu The cross context virtual CPU structure.
14319 * @param fWillExit To be defined.
14320 * @param cMinInstructions Minimum number of instructions to execute before checking for FFs.
14321 * @param cMaxInstructions Maximum number of instructions to execute.
14322 * @param cMaxInstructionsWithoutExits
14323 * The max number of instructions without exits.
14324 * @param pStats Where to return statistics.
14325 */
14326VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPU pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
14327 uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
14328{
14329 NOREF(fWillExit); /** @todo define flexible exit crits */
14330
14331 /*
14332 * Initialize return stats.
14333 */
14334 pStats->cInstructions = 0;
14335 pStats->cExits = 0;
14336 pStats->cMaxExitDistance = 0;
14337 pStats->cReserved = 0;
14338
14339 /*
14340 * Initial decoder init w/ prefetch, then setup setjmp.
14341 */
14342 VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false);
14343 if (rcStrict == VINF_SUCCESS)
14344 {
14345#ifdef IEM_WITH_SETJMP
14346 jmp_buf JmpBuf;
14347 jmp_buf *pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf);
14348 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf;
14349 pVCpu->iem.s.cActiveMappings = 0;
14350 if ((rcStrict = setjmp(JmpBuf)) == 0)
14351#endif
14352 {
14353#ifdef IN_RING0
14354 bool const fCheckPreemptionPending = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
14355#endif
14356 uint32_t cInstructionSinceLastExit = 0;
14357
14358 /*
14359 * The run loop. We limit ourselves to 4096 instructions right now.
14360 */
14361 PVM pVM = pVCpu->CTX_SUFF(pVM);
14362 for (;;)
14363 {
14364 /*
14365 * Log the state.
14366 */
14367#ifdef LOG_ENABLED
14368 iemLogCurInstr(pVCpu, true, "IEMExecForExits");
14369#endif
14370
14371 /*
14372 * Do the decoding and emulation.
14373 */
14374 uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
14375
14376 uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
14377 rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
14378
14379 if ( cPotentialExits != pVCpu->iem.s.cPotentialExits
14380 && cInstructionSinceLastExit > 0 /* don't count the first */ )
14381 {
14382 pStats->cExits += 1;
14383 if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
14384 pStats->cMaxExitDistance = cInstructionSinceLastExit;
14385 cInstructionSinceLastExit = 0;
14386 }
14387
14388 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
14389 {
14390 Assert(pVCpu->iem.s.cActiveMappings == 0);
14391 pVCpu->iem.s.cInstructions++;
14392 pStats->cInstructions++;
14393 cInstructionSinceLastExit++;
14394 if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
14395 {
14396 uint32_t fCpu = pVCpu->fLocalForcedActions
14397 & ( VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
14398 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
14399 | VMCPU_FF_TLB_FLUSH
14400#ifdef VBOX_WITH_RAW_MODE
14401 | VMCPU_FF_TRPM_SYNC_IDT
14402 | VMCPU_FF_SELM_SYNC_TSS
14403 | VMCPU_FF_SELM_SYNC_GDT
14404 | VMCPU_FF_SELM_SYNC_LDT
14405#endif
14406 | VMCPU_FF_INHIBIT_INTERRUPTS
14407 | VMCPU_FF_BLOCK_NMIS
14408 | VMCPU_FF_UNHALT ));
14409
14410 if (RT_LIKELY( ( ( !fCpu
14411 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
14412 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
14413 && !VM_FF_IS_PENDING(pVM, VM_FF_ALL_MASK) )
14414 || pStats->cInstructions < cMinInstructions))
14415 {
14416 if (pStats->cInstructions < cMaxInstructions)
14417 {
14418 if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
14419 {
14420#ifdef IN_RING0
14421 if ( !fCheckPreemptionPending
14422 || !RTThreadPreemptIsPending(NIL_RTTHREAD))
14423#endif
14424 {
14425 Assert(pVCpu->iem.s.cActiveMappings == 0);
14426 iemReInitDecoder(pVCpu);
14427 continue;
14428 }
14429#ifdef IN_RING0
14430 rcStrict = VINF_EM_RAW_INTERRUPT;
14431 break;
14432#endif
14433 }
14434 }
14435 }
14436 Assert(!(fCpu & VMCPU_FF_IEM));
14437 }
14438 Assert(pVCpu->iem.s.cActiveMappings == 0);
14439 }
14440 else if (pVCpu->iem.s.cActiveMappings > 0)
14441 iemMemRollback(pVCpu);
14442 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14443 break;
14444 }
14445 }
14446#ifdef IEM_WITH_SETJMP
14447 else
14448 {
14449 if (pVCpu->iem.s.cActiveMappings > 0)
14450 iemMemRollback(pVCpu);
14451 pVCpu->iem.s.cLongJumps++;
14452 }
14453 pVCpu->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf;
14454#endif
14455
14456 /*
14457 * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
14458 */
14459 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
14460 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
14461 }
14462 else
14463 {
14464 if (pVCpu->iem.s.cActiveMappings > 0)
14465 iemMemRollback(pVCpu);
14466
14467#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
14468 /*
14469 * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
14470 * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
14471 */
14472 rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
14473#endif
14474 }
14475
14476 /*
14477 * Maybe re-enter raw-mode and log.
14478 */
14479#ifdef IN_RC
14480 rcStrict = iemRCRawMaybeReenter(pVCpu, rcStrict);
14481#endif
14482 if (rcStrict != VINF_SUCCESS)
14483 LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
14484 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
14485 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
14486 return rcStrict;
14487}
14488
14489
14490/**
14491 * Injects a trap, fault, abort, software interrupt or external interrupt.
14492 *
14493 * The parameter list matches TRPMQueryTrapAll pretty closely.
14494 *
14495 * @returns Strict VBox status code.
14496 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14497 * @param u8TrapNo The trap number.
14498 * @param enmType What type is it (trap/fault/abort), software
14499 * interrupt or hardware interrupt.
14500 * @param uErrCode The error code if applicable.
14501 * @param uCr2 The CR2 value if applicable.
14502 * @param cbInstr The instruction length (only relevant for
14503 * software interrupts).
14504 */
14505VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPU pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
14506 uint8_t cbInstr)
14507{
14508 iemInitDecoder(pVCpu, false);
14509#ifdef DBGFTRACE_ENABLED
14510 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
14511 u8TrapNo, enmType, uErrCode, uCr2);
14512#endif
14513
14514 uint32_t fFlags;
14515 switch (enmType)
14516 {
14517 case TRPM_HARDWARE_INT:
14518 Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
14519 fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
14520 uErrCode = uCr2 = 0;
14521 break;
14522
14523 case TRPM_SOFTWARE_INT:
14524 Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
14525 fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
14526 uErrCode = uCr2 = 0;
14527 break;
14528
14529 case TRPM_TRAP:
14530 Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
14531 fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
14532 if (u8TrapNo == X86_XCPT_PF)
14533 fFlags |= IEM_XCPT_FLAGS_CR2;
14534 switch (u8TrapNo)
14535 {
14536 case X86_XCPT_DF:
14537 case X86_XCPT_TS:
14538 case X86_XCPT_NP:
14539 case X86_XCPT_SS:
14540 case X86_XCPT_PF:
14541 case X86_XCPT_AC:
14542 fFlags |= IEM_XCPT_FLAGS_ERR;
14543 break;
14544
14545 case X86_XCPT_NMI:
14546 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
14547 break;
14548 }
14549 break;
14550
14551 IEM_NOT_REACHED_DEFAULT_CASE_RET();
14552 }
14553
14554 VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
14555
14556 if (pVCpu->iem.s.cActiveMappings > 0)
14557 iemMemRollback(pVCpu);
14558
14559 return rcStrict;
14560}
14561
14562
14563/**
14564 * Injects the active TRPM event.
14565 *
14566 * @returns Strict VBox status code.
14567 * @param pVCpu The cross context virtual CPU structure.
14568 */
14569VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPU pVCpu)
14570{
14571#ifndef IEM_IMPLEMENTS_TASKSWITCH
14572 IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
14573#else
14574 uint8_t u8TrapNo;
14575 TRPMEVENT enmType;
14576 RTGCUINT uErrCode;
14577 RTGCUINTPTR uCr2;
14578 uint8_t cbInstr;
14579 int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr);
14580 if (RT_FAILURE(rc))
14581 return rc;
14582
14583 VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
14584# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
14585 if (rcStrict == VINF_SVM_VMEXIT)
14586 rcStrict = VINF_SUCCESS;
14587# endif
14588
14589 /** @todo Are there any other codes that imply the event was successfully
14590 * delivered to the guest? See @bugref{6607}. */
14591 if ( rcStrict == VINF_SUCCESS
14592 || rcStrict == VINF_IEM_RAISED_XCPT)
14593 TRPMResetTrap(pVCpu);
14594
14595 return rcStrict;
14596#endif
14597}
14598
14599
14600VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
14601{
14602 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
14603 return VERR_NOT_IMPLEMENTED;
14604}
14605
14606
14607VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
14608{
14609 RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
14610 return VERR_NOT_IMPLEMENTED;
14611}
14612
14613
14614#if 0 /* The IRET-to-v8086 mode in PATM is very optimistic, so I don't dare do this yet. */
14615/**
14616 * Executes a IRET instruction with default operand size.
14617 *
14618 * This is for PATM.
14619 *
14620 * @returns VBox status code.
14621 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
14622 * @param pCtxCore The register frame.
14623 */
14624VMM_INT_DECL(int) IEMExecInstr_iret(PVMCPU pVCpu, PCPUMCTXCORE pCtxCore)
14625{
14626 PCPUMCTX pCtx = IEM_GET_CTX(pVCpu);
14627
14628 iemCtxCoreToCtx(pCtx, pCtxCore);
14629 iemInitDecoder(pVCpu);
14630 VBOXSTRICTRC rcStrict = iemCImpl_iret(pVCpu, 1, pVCpu->iem.s.enmDefOpSize);
14631 if (rcStrict == VINF_SUCCESS)
14632 iemCtxToCtxCore(pCtxCore, pCtx);
14633 else
14634 LogFlow(("IEMExecInstr_iret: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
14635 pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
14636 return rcStrict;
14637}
14638#endif
14639
14640
14641/**
14642 * Macro used by the IEMExec* method to check the given instruction length.
14643 *
14644 * Will return on failure!
14645 *
14646 * @param a_cbInstr The given instruction length.
14647 * @param a_cbMin The minimum length.
14648 */
14649#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
14650 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
14651 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
14652
14653
14654/**
14655 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
14656 *
14657 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
14658 *
14659 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
14660 * @param pVCpu The cross context virtual CPU structure of the calling thread.
14661 * @param rcStrict The status code to fiddle.
14662 */
14663DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
14664{
14665 iemUninitExec(pVCpu);
14666#ifdef IN_RC
14667 return iemRCRawMaybeReenter(pVCpu, iemExecStatusCodeFiddling(pVCpu, rcStrict));
14668#else
14669 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
14670#endif
14671}
14672
14673
14674/**
14675 * Interface for HM and EM for executing string I/O OUT (write) instructions.
14676 *
14677 * This API ASSUMES that the caller has already verified that the guest code is
14678 * allowed to access the I/O port. (The I/O port is in the DX register in the
14679 * guest state.)
14680 *
14681 * @returns Strict VBox status code.
14682 * @param pVCpu The cross context virtual CPU structure.
14683 * @param cbValue The size of the I/O port access (1, 2, or 4).
14684 * @param enmAddrMode The addressing mode.
14685 * @param fRepPrefix Indicates whether a repeat prefix is used
14686 * (doesn't matter which for this instruction).
14687 * @param cbInstr The instruction length in bytes.
14688 * @param iEffSeg The effective segment address.
14689 * @param fIoChecked Whether the access to the I/O port has been
14690 * checked or not. It's typically checked in the
14691 * HM scenario.
14692 */
14693VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPU pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
14694 bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
14695{
14696 AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
14697 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14698
14699 /*
14700 * State init.
14701 */
14702 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14703
14704 /*
14705 * Switch orgy for getting to the right handler.
14706 */
14707 VBOXSTRICTRC rcStrict;
14708 if (fRepPrefix)
14709 {
14710 switch (enmAddrMode)
14711 {
14712 case IEMMODE_16BIT:
14713 switch (cbValue)
14714 {
14715 case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14716 case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14717 case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14718 default:
14719 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14720 }
14721 break;
14722
14723 case IEMMODE_32BIT:
14724 switch (cbValue)
14725 {
14726 case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14727 case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14728 case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14729 default:
14730 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14731 }
14732 break;
14733
14734 case IEMMODE_64BIT:
14735 switch (cbValue)
14736 {
14737 case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14738 case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14739 case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14740 default:
14741 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14742 }
14743 break;
14744
14745 default:
14746 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14747 }
14748 }
14749 else
14750 {
14751 switch (enmAddrMode)
14752 {
14753 case IEMMODE_16BIT:
14754 switch (cbValue)
14755 {
14756 case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14757 case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14758 case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14759 default:
14760 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14761 }
14762 break;
14763
14764 case IEMMODE_32BIT:
14765 switch (cbValue)
14766 {
14767 case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14768 case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14769 case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14770 default:
14771 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14772 }
14773 break;
14774
14775 case IEMMODE_64BIT:
14776 switch (cbValue)
14777 {
14778 case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14779 case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14780 case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
14781 default:
14782 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14783 }
14784 break;
14785
14786 default:
14787 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14788 }
14789 }
14790
14791 if (pVCpu->iem.s.cActiveMappings)
14792 iemMemRollback(pVCpu);
14793
14794 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14795}
14796
14797
14798/**
14799 * Interface for HM and EM for executing string I/O IN (read) instructions.
14800 *
14801 * This API ASSUMES that the caller has already verified that the guest code is
14802 * allowed to access the I/O port. (The I/O port is in the DX register in the
14803 * guest state.)
14804 *
14805 * @returns Strict VBox status code.
14806 * @param pVCpu The cross context virtual CPU structure.
14807 * @param cbValue The size of the I/O port access (1, 2, or 4).
14808 * @param enmAddrMode The addressing mode.
14809 * @param fRepPrefix Indicates whether a repeat prefix is used
14810 * (doesn't matter which for this instruction).
14811 * @param cbInstr The instruction length in bytes.
14812 * @param fIoChecked Whether the access to the I/O port has been
14813 * checked or not. It's typically checked in the
14814 * HM scenario.
14815 */
14816VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPU pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
14817 bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
14818{
14819 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14820
14821 /*
14822 * State init.
14823 */
14824 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14825
14826 /*
14827 * Switch orgy for getting to the right handler.
14828 */
14829 VBOXSTRICTRC rcStrict;
14830 if (fRepPrefix)
14831 {
14832 switch (enmAddrMode)
14833 {
14834 case IEMMODE_16BIT:
14835 switch (cbValue)
14836 {
14837 case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
14838 case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
14839 case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
14840 default:
14841 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14842 }
14843 break;
14844
14845 case IEMMODE_32BIT:
14846 switch (cbValue)
14847 {
14848 case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
14849 case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
14850 case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
14851 default:
14852 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14853 }
14854 break;
14855
14856 case IEMMODE_64BIT:
14857 switch (cbValue)
14858 {
14859 case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
14860 case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
14861 case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
14862 default:
14863 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14864 }
14865 break;
14866
14867 default:
14868 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14869 }
14870 }
14871 else
14872 {
14873 switch (enmAddrMode)
14874 {
14875 case IEMMODE_16BIT:
14876 switch (cbValue)
14877 {
14878 case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
14879 case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
14880 case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
14881 default:
14882 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14883 }
14884 break;
14885
14886 case IEMMODE_32BIT:
14887 switch (cbValue)
14888 {
14889 case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
14890 case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
14891 case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
14892 default:
14893 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14894 }
14895 break;
14896
14897 case IEMMODE_64BIT:
14898 switch (cbValue)
14899 {
14900 case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
14901 case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
14902 case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
14903 default:
14904 AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
14905 }
14906 break;
14907
14908 default:
14909 AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
14910 }
14911 }
14912
14913 Assert(pVCpu->iem.s.cActiveMappings == 0 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
14914 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14915}
14916
14917
14918/**
14919 * Interface for rawmode to write execute an OUT instruction.
14920 *
14921 * @returns Strict VBox status code.
14922 * @param pVCpu The cross context virtual CPU structure.
14923 * @param cbInstr The instruction length in bytes.
14924 * @param u16Port The port to read.
14925 * @param cbReg The register size.
14926 *
14927 * @remarks In ring-0 not all of the state needs to be synced in.
14928 */
14929VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPU pVCpu, uint8_t cbInstr, uint16_t u16Port, uint8_t cbReg)
14930{
14931 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14932 Assert(cbReg <= 4 && cbReg != 3);
14933
14934 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14935 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_out, u16Port, cbReg);
14936 Assert(!pVCpu->iem.s.cActiveMappings);
14937 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14938}
14939
14940
14941/**
14942 * Interface for rawmode to write execute an IN instruction.
14943 *
14944 * @returns Strict VBox status code.
14945 * @param pVCpu The cross context virtual CPU structure.
14946 * @param cbInstr The instruction length in bytes.
14947 * @param u16Port The port to read.
14948 * @param cbReg The register size.
14949 */
14950VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPU pVCpu, uint8_t cbInstr, uint16_t u16Port, uint8_t cbReg)
14951{
14952 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
14953 Assert(cbReg <= 4 && cbReg != 3);
14954
14955 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14956 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_in, u16Port, cbReg);
14957 Assert(!pVCpu->iem.s.cActiveMappings);
14958 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14959}
14960
14961
14962/**
14963 * Interface for HM and EM to write to a CRx register.
14964 *
14965 * @returns Strict VBox status code.
14966 * @param pVCpu The cross context virtual CPU structure.
14967 * @param cbInstr The instruction length in bytes.
14968 * @param iCrReg The control register number (destination).
14969 * @param iGReg The general purpose register number (source).
14970 *
14971 * @remarks In ring-0 not all of the state needs to be synced in.
14972 */
14973VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPU pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
14974{
14975 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
14976 Assert(iCrReg < 16);
14977 Assert(iGReg < 16);
14978
14979 iemInitExec(pVCpu, false /*fBypassHandlers*/);
14980 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
14981 Assert(!pVCpu->iem.s.cActiveMappings);
14982 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
14983}
14984
14985
14986/**
14987 * Interface for HM and EM to read from a CRx register.
14988 *
14989 * @returns Strict VBox status code.
14990 * @param pVCpu The cross context virtual CPU structure.
14991 * @param cbInstr The instruction length in bytes.
14992 * @param iGReg The general purpose register number (destination).
14993 * @param iCrReg The control register number (source).
14994 *
14995 * @remarks In ring-0 not all of the state needs to be synced in.
14996 */
14997VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPU pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
14998{
14999 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15000 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
15001 | CPUMCTX_EXTRN_APIC_TPR);
15002 Assert(iCrReg < 16);
15003 Assert(iGReg < 16);
15004
15005 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15006 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
15007 Assert(!pVCpu->iem.s.cActiveMappings);
15008 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15009}
15010
15011
15012/**
15013 * Interface for HM and EM to clear the CR0[TS] bit.
15014 *
15015 * @returns Strict VBox status code.
15016 * @param pVCpu The cross context virtual CPU structure.
15017 * @param cbInstr The instruction length in bytes.
15018 *
15019 * @remarks In ring-0 not all of the state needs to be synced in.
15020 */
15021VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPU pVCpu, uint8_t cbInstr)
15022{
15023 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15024
15025 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15026 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
15027 Assert(!pVCpu->iem.s.cActiveMappings);
15028 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15029}
15030
15031
15032/**
15033 * Interface for HM and EM to emulate the LMSW instruction (loads CR0).
15034 *
15035 * @returns Strict VBox status code.
15036 * @param pVCpu The cross context virtual CPU structure.
15037 * @param cbInstr The instruction length in bytes.
15038 * @param uValue The value to load into CR0.
15039 *
15040 * @remarks In ring-0 not all of the state needs to be synced in.
15041 */
15042VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPU pVCpu, uint8_t cbInstr, uint16_t uValue)
15043{
15044 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15045
15046 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15047 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_lmsw, uValue);
15048 Assert(!pVCpu->iem.s.cActiveMappings);
15049 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15050}
15051
15052
15053/**
15054 * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
15055 *
15056 * Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
15057 *
15058 * @returns Strict VBox status code.
15059 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15060 * @param cbInstr The instruction length in bytes.
15061 * @remarks In ring-0 not all of the state needs to be synced in.
15062 * @thread EMT(pVCpu)
15063 */
15064VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPU pVCpu, uint8_t cbInstr)
15065{
15066 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15067
15068 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15069 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
15070 Assert(!pVCpu->iem.s.cActiveMappings);
15071 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15072}
15073
15074
15075/**
15076 * Interface for HM and EM to emulate the WBINVD instruction.
15077 *
15078 * @returns Strict VBox status code.
15079 * @param pVCpu The cross context virtual CPU structure.
15080 * @param cbInstr The instruction length in bytes.
15081 *
15082 * @remarks In ring-0 not all of the state needs to be synced in.
15083 */
15084VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPU pVCpu, uint8_t cbInstr)
15085{
15086 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15087
15088 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15089 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
15090 Assert(!pVCpu->iem.s.cActiveMappings);
15091 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15092}
15093
15094
15095/**
15096 * Interface for HM and EM to emulate the INVD instruction.
15097 *
15098 * @returns Strict VBox status code.
15099 * @param pVCpu The cross context virtual CPU structure.
15100 * @param cbInstr The instruction length in bytes.
15101 *
15102 * @remarks In ring-0 not all of the state needs to be synced in.
15103 */
15104VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPU pVCpu, uint8_t cbInstr)
15105{
15106 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15107
15108 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15109 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
15110 Assert(!pVCpu->iem.s.cActiveMappings);
15111 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15112}
15113
15114
15115/**
15116 * Interface for HM and EM to emulate the INVLPG instruction.
15117 *
15118 * @returns Strict VBox status code.
15119 * @retval VINF_PGM_SYNC_CR3
15120 *
15121 * @param pVCpu The cross context virtual CPU structure.
15122 * @param cbInstr The instruction length in bytes.
15123 * @param GCPtrPage The effective address of the page to invalidate.
15124 *
15125 * @remarks In ring-0 not all of the state needs to be synced in.
15126 */
15127VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPU pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
15128{
15129 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15130
15131 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15132 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
15133 Assert(!pVCpu->iem.s.cActiveMappings);
15134 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15135}
15136
15137
15138/**
15139 * Interface for HM and EM to emulate the CPUID instruction.
15140 *
15141 * @returns Strict VBox status code.
15142 *
15143 * @param pVCpu The cross context virtual CPU structure.
15144 * @param cbInstr The instruction length in bytes.
15145 *
15146 * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
15147 */
15148VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPU pVCpu, uint8_t cbInstr)
15149{
15150 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15151 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
15152
15153 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15154 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
15155 Assert(!pVCpu->iem.s.cActiveMappings);
15156 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15157}
15158
15159
15160/**
15161 * Interface for HM and EM to emulate the RDPMC instruction.
15162 *
15163 * @returns Strict VBox status code.
15164 *
15165 * @param pVCpu The cross context virtual CPU structure.
15166 * @param cbInstr The instruction length in bytes.
15167 *
15168 * @remarks Not all of the state needs to be synced in.
15169 */
15170VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPU pVCpu, uint8_t cbInstr)
15171{
15172 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15173 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
15174
15175 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15176 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
15177 Assert(!pVCpu->iem.s.cActiveMappings);
15178 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15179}
15180
15181
15182/**
15183 * Interface for HM and EM to emulate the RDTSC instruction.
15184 *
15185 * @returns Strict VBox status code.
15186 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15187 *
15188 * @param pVCpu The cross context virtual CPU structure.
15189 * @param cbInstr The instruction length in bytes.
15190 *
15191 * @remarks Not all of the state needs to be synced in.
15192 */
15193VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPU pVCpu, uint8_t cbInstr)
15194{
15195 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15196 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
15197
15198 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15199 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
15200 Assert(!pVCpu->iem.s.cActiveMappings);
15201 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15202}
15203
15204
15205/**
15206 * Interface for HM and EM to emulate the RDTSCP instruction.
15207 *
15208 * @returns Strict VBox status code.
15209 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15210 *
15211 * @param pVCpu The cross context virtual CPU structure.
15212 * @param cbInstr The instruction length in bytes.
15213 *
15214 * @remarks Not all of the state needs to be synced in. Recommended
15215 * to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
15216 */
15217VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPU pVCpu, uint8_t cbInstr)
15218{
15219 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15220 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
15221
15222 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15223 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
15224 Assert(!pVCpu->iem.s.cActiveMappings);
15225 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15226}
15227
15228
15229/**
15230 * Interface for HM and EM to emulate the RDMSR instruction.
15231 *
15232 * @returns Strict VBox status code.
15233 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15234 *
15235 * @param pVCpu The cross context virtual CPU structure.
15236 * @param cbInstr The instruction length in bytes.
15237 *
15238 * @remarks Not all of the state needs to be synced in. Requires RCX and
15239 * (currently) all MSRs.
15240 */
15241VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPU pVCpu, uint8_t cbInstr)
15242{
15243 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15244 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS);
15245
15246 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15247 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
15248 Assert(!pVCpu->iem.s.cActiveMappings);
15249 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15250}
15251
15252
15253/**
15254 * Interface for HM and EM to emulate the WRMSR instruction.
15255 *
15256 * @returns Strict VBox status code.
15257 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15258 *
15259 * @param pVCpu The cross context virtual CPU structure.
15260 * @param cbInstr The instruction length in bytes.
15261 *
15262 * @remarks Not all of the state needs to be synced in. Requires RCX, RAX, RDX,
15263 * and (currently) all MSRs.
15264 */
15265VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPU pVCpu, uint8_t cbInstr)
15266{
15267 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
15268 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
15269 | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS);
15270
15271 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15272 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
15273 Assert(!pVCpu->iem.s.cActiveMappings);
15274 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15275}
15276
15277
15278/**
15279 * Interface for HM and EM to emulate the MONITOR instruction.
15280 *
15281 * @returns Strict VBox status code.
15282 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15283 *
15284 * @param pVCpu The cross context virtual CPU structure.
15285 * @param cbInstr The instruction length in bytes.
15286 *
15287 * @remarks Not all of the state needs to be synced in.
15288 * @remarks ASSUMES the default segment of DS and no segment override prefixes
15289 * are used.
15290 */
15291VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPU pVCpu, uint8_t cbInstr)
15292{
15293 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15294 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
15295
15296 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15297 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
15298 Assert(!pVCpu->iem.s.cActiveMappings);
15299 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15300}
15301
15302
15303/**
15304 * Interface for HM and EM to emulate the MWAIT instruction.
15305 *
15306 * @returns Strict VBox status code.
15307 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15308 *
15309 * @param pVCpu The cross context virtual CPU structure.
15310 * @param cbInstr The instruction length in bytes.
15311 *
15312 * @remarks Not all of the state needs to be synced in.
15313 */
15314VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPU pVCpu, uint8_t cbInstr)
15315{
15316 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15317
15318 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15319 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
15320 Assert(!pVCpu->iem.s.cActiveMappings);
15321 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15322}
15323
15324
15325/**
15326 * Interface for HM and EM to emulate the HLT instruction.
15327 *
15328 * @returns Strict VBox status code.
15329 * @retval VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
15330 *
15331 * @param pVCpu The cross context virtual CPU structure.
15332 * @param cbInstr The instruction length in bytes.
15333 *
15334 * @remarks Not all of the state needs to be synced in.
15335 */
15336VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPU pVCpu, uint8_t cbInstr)
15337{
15338 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
15339
15340 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15341 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
15342 Assert(!pVCpu->iem.s.cActiveMappings);
15343 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15344}
15345
15346
15347/**
15348 * Checks if IEM is in the process of delivering an event (interrupt or
15349 * exception).
15350 *
15351 * @returns true if we're in the process of raising an interrupt or exception,
15352 * false otherwise.
15353 * @param pVCpu The cross context virtual CPU structure.
15354 * @param puVector Where to store the vector associated with the
15355 * currently delivered event, optional.
15356 * @param pfFlags Where to store th event delivery flags (see
15357 * IEM_XCPT_FLAGS_XXX), optional.
15358 * @param puErr Where to store the error code associated with the
15359 * event, optional.
15360 * @param puCr2 Where to store the CR2 associated with the event,
15361 * optional.
15362 * @remarks The caller should check the flags to determine if the error code and
15363 * CR2 are valid for the event.
15364 */
15365VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPU pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2)
15366{
15367 bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
15368 if (fRaisingXcpt)
15369 {
15370 if (puVector)
15371 *puVector = pVCpu->iem.s.uCurXcpt;
15372 if (pfFlags)
15373 *pfFlags = pVCpu->iem.s.fCurXcpt;
15374 if (puErr)
15375 *puErr = pVCpu->iem.s.uCurXcptErr;
15376 if (puCr2)
15377 *puCr2 = pVCpu->iem.s.uCurXcptCr2;
15378 }
15379 return fRaisingXcpt;
15380}
15381
15382#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
15383
15384/**
15385 * Interface for HM and EM to emulate the CLGI instruction.
15386 *
15387 * @returns Strict VBox status code.
15388 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15389 * @param cbInstr The instruction length in bytes.
15390 * @thread EMT(pVCpu)
15391 */
15392VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClgi(PVMCPU pVCpu, uint8_t cbInstr)
15393{
15394 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15395
15396 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15397 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clgi);
15398 Assert(!pVCpu->iem.s.cActiveMappings);
15399 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15400}
15401
15402
15403/**
15404 * Interface for HM and EM to emulate the STGI instruction.
15405 *
15406 * @returns Strict VBox status code.
15407 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15408 * @param cbInstr The instruction length in bytes.
15409 * @thread EMT(pVCpu)
15410 */
15411VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedStgi(PVMCPU pVCpu, uint8_t cbInstr)
15412{
15413 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15414
15415 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15416 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_stgi);
15417 Assert(!pVCpu->iem.s.cActiveMappings);
15418 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15419}
15420
15421
15422/**
15423 * Interface for HM and EM to emulate the VMLOAD instruction.
15424 *
15425 * @returns Strict VBox status code.
15426 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15427 * @param cbInstr The instruction length in bytes.
15428 * @thread EMT(pVCpu)
15429 */
15430VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmload(PVMCPU pVCpu, uint8_t cbInstr)
15431{
15432 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15433
15434 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15435 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmload);
15436 Assert(!pVCpu->iem.s.cActiveMappings);
15437 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15438}
15439
15440
15441/**
15442 * Interface for HM and EM to emulate the VMSAVE instruction.
15443 *
15444 * @returns Strict VBox status code.
15445 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15446 * @param cbInstr The instruction length in bytes.
15447 * @thread EMT(pVCpu)
15448 */
15449VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmsave(PVMCPU pVCpu, uint8_t cbInstr)
15450{
15451 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15452
15453 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15454 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmsave);
15455 Assert(!pVCpu->iem.s.cActiveMappings);
15456 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15457}
15458
15459
15460/**
15461 * Interface for HM and EM to emulate the INVLPGA instruction.
15462 *
15463 * @returns Strict VBox status code.
15464 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15465 * @param cbInstr The instruction length in bytes.
15466 * @thread EMT(pVCpu)
15467 */
15468VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpga(PVMCPU pVCpu, uint8_t cbInstr)
15469{
15470 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15471
15472 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15473 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invlpga);
15474 Assert(!pVCpu->iem.s.cActiveMappings);
15475 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15476}
15477
15478
15479/**
15480 * Interface for HM and EM to emulate the VMRUN instruction.
15481 *
15482 * @returns Strict VBox status code.
15483 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15484 * @param cbInstr The instruction length in bytes.
15485 * @thread EMT(pVCpu)
15486 */
15487VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmrun(PVMCPU pVCpu, uint8_t cbInstr)
15488{
15489 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15490 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK);
15491
15492 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15493 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmrun);
15494 Assert(!pVCpu->iem.s.cActiveMappings);
15495 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15496}
15497
15498
15499/**
15500 * Interface for HM and EM to emulate \#VMEXIT.
15501 *
15502 * @returns Strict VBox status code.
15503 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15504 * @param uExitCode The exit code.
15505 * @param uExitInfo1 The exit info. 1 field.
15506 * @param uExitInfo2 The exit info. 2 field.
15507 * @thread EMT(pVCpu)
15508 */
15509VMM_INT_DECL(VBOXSTRICTRC) IEMExecSvmVmexit(PVMCPU pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2)
15510{
15511 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
15512 VBOXSTRICTRC rcStrict = iemSvmVmexit(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
15513 if (pVCpu->iem.s.cActiveMappings)
15514 iemMemRollback(pVCpu);
15515 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15516}
15517
15518#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
15519
15520#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15521
15522/**
15523 * Interface for HM and EM to emulate the VMXOFF instruction.
15524 *
15525 * @returns Strict VBox status code.
15526 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15527 * @param cbInstr The instruction length in bytes.
15528 * @thread EMT(pVCpu)
15529 */
15530VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxoff(PVMCPU pVCpu, uint8_t cbInstr)
15531{
15532 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15533
15534 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15535 VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmxoff);
15536 Assert(!pVCpu->iem.s.cActiveMappings);
15537 return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
15538}
15539
15540
15541/**
15542 * Interface for HM and EM to emulate the VMXON instruction.
15543 *
15544 * @returns Strict VBox status code.
15545 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15546 * @param cbInstr The instruction length in bytes.
15547 * @param GCPtrVmxon The linear address of the VMXON pointer.
15548 * @param uExitInstrInfo The VM-exit instruction information field.
15549 * @param GCPtrDisp The displacement field for @a GCPtrVmxon if any.
15550 * @thread EMT(pVCpu)
15551 */
15552VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxon(PVMCPU pVCpu, uint8_t cbInstr, RTGCPHYS GCPtrVmxon, uint32_t uExitInstrInfo,
15553 RTGCPTR GCPtrDisp)
15554{
15555 IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
15556 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT);
15557
15558 iemInitExec(pVCpu, false /*fBypassHandlers*/);
15559 PCVMXEXITINSTRINFO pExitInstrInfo = (PCVMXEXITINSTRINFO)&uExitInstrInfo;
15560 VBOXSTRICTRC rcStrict = iemVmxVmxon(pVCpu, cbInstr, GCPtrVmxon, pExitInstrInfo, GCPtrDisp);
15561 if (pVCpu->iem.s.cActiveMappings)
15562 iemMemRollback(pVCpu);
15563 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
15564}
15565
15566#endif
15567
15568#ifdef IN_RING3
15569
15570/**
15571 * Handles the unlikely and probably fatal merge cases.
15572 *
15573 * @returns Merged status code.
15574 * @param rcStrict Current EM status code.
15575 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
15576 * with @a rcStrict.
15577 * @param iMemMap The memory mapping index. For error reporting only.
15578 * @param pVCpu The cross context virtual CPU structure of the calling
15579 * thread, for error reporting only.
15580 */
15581DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
15582 unsigned iMemMap, PVMCPU pVCpu)
15583{
15584 if (RT_FAILURE_NP(rcStrict))
15585 return rcStrict;
15586
15587 if (RT_FAILURE_NP(rcStrictCommit))
15588 return rcStrictCommit;
15589
15590 if (rcStrict == rcStrictCommit)
15591 return rcStrictCommit;
15592
15593 AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
15594 VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
15595 pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
15596 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
15597 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
15598 return VERR_IOM_FF_STATUS_IPE;
15599}
15600
15601
15602/**
15603 * Helper for IOMR3ProcessForceFlag.
15604 *
15605 * @returns Merged status code.
15606 * @param rcStrict Current EM status code.
15607 * @param rcStrictCommit The IOM I/O or MMIO write commit status to merge
15608 * with @a rcStrict.
15609 * @param iMemMap The memory mapping index. For error reporting only.
15610 * @param pVCpu The cross context virtual CPU structure of the calling
15611 * thread, for error reporting only.
15612 */
15613DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPU pVCpu)
15614{
15615 /* Simple. */
15616 if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))
15617 return rcStrictCommit;
15618
15619 if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
15620 return rcStrict;
15621
15622 /* EM scheduling status codes. */
15623 if (RT_LIKELY( rcStrict >= VINF_EM_FIRST
15624 && rcStrict <= VINF_EM_LAST))
15625 {
15626 if (RT_LIKELY( rcStrictCommit >= VINF_EM_FIRST
15627 && rcStrictCommit <= VINF_EM_LAST))
15628 return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
15629 }
15630
15631 /* Unlikely */
15632 return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
15633}
15634
15635
15636/**
15637 * Called by force-flag handling code when VMCPU_FF_IEM is set.
15638 *
15639 * @returns Merge between @a rcStrict and what the commit operation returned.
15640 * @param pVM The cross context VM structure.
15641 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
15642 * @param rcStrict The status code returned by ring-0 or raw-mode.
15643 */
15644VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rcStrict)
15645{
15646 /*
15647 * Reset the pending commit.
15648 */
15649 AssertMsg( (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)
15650 & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),
15651 ("%#x %#x %#x\n",
15652 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
15653 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
15654
15655 /*
15656 * Commit the pending bounce buffers (usually just one).
15657 */
15658 unsigned cBufs = 0;
15659 unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
15660 while (iMemMap-- > 0)
15661 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))
15662 {
15663 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
15664 Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
15665 Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
15666
15667 uint16_t const cbFirst = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
15668 uint16_t const cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
15669 uint8_t const *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
15670
15671 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
15672 {
15673 VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
15674 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
15675 pbBuf,
15676 cbFirst,
15677 PGMACCESSORIGIN_IEM);
15678 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
15679 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
15680 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
15681 VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
15682 }
15683
15684 if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
15685 {
15686 VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
15687 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
15688 pbBuf + cbFirst,
15689 cbSecond,
15690 PGMACCESSORIGIN_IEM);
15691 rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
15692 Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
15693 iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
15694 VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
15695 }
15696 cBufs++;
15697 pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
15698 }
15699
15700 AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
15701 ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
15702 pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
15703 pVCpu->iem.s.cActiveMappings = 0;
15704 return rcStrict;
15705}
15706
15707#endif /* IN_RING3 */
15708
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette