VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 100746

Last change on this file since 100746 was 100734, checked in by vboxsync, 17 months ago

VMM/IEM: Generate TBs for invalid instruction encodings as well. This involved special casing recompiler call generation for C instruction implementation function that doesn't take any extra arguments, so that we can catch all the deeply hidden IEMOP_RAISE_INVALID_OPCODE_RET invocations and similar. Also had to clean up hacky decoding of effective address related opcode bytes for undefined opcodes, introducing IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) to hide the uglyness. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 119.8 KB
Line 
1/* $Id: IEMInline.h 100734 2023-07-29 02:04:22Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Inlined Functions.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29#define VMM_INCLUDED_SRC_include_IEMInline_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35
36/**
37 * Makes status code addjustments (pass up from I/O and access handler)
38 * as well as maintaining statistics.
39 *
40 * @returns Strict VBox status code to pass up.
41 * @param pVCpu The cross context virtual CPU structure of the calling thread.
42 * @param rcStrict The status from executing an instruction.
43 */
44DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45{
46 if (rcStrict != VINF_SUCCESS)
47 {
48 if (RT_SUCCESS(rcStrict))
49 {
50 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51 || rcStrict == VINF_IOM_R3_IOPORT_READ
52 || rcStrict == VINF_IOM_R3_IOPORT_WRITE
53 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54 || rcStrict == VINF_IOM_R3_MMIO_READ
55 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56 || rcStrict == VINF_IOM_R3_MMIO_WRITE
57 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58 || rcStrict == VINF_CPUM_R3_MSR_READ
59 || rcStrict == VINF_CPUM_R3_MSR_WRITE
60 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
61 || rcStrict == VINF_EM_RAW_TO_R3
62 || rcStrict == VINF_EM_TRIPLE_FAULT
63 || rcStrict == VINF_GIM_R3_HYPERCALL
64 /* raw-mode / virt handlers only: */
65 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69 || rcStrict == VINF_SELM_SYNC_GDT
70 || rcStrict == VINF_CSAM_PENDING_ACTION
71 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72 /* nested hw.virt codes: */
73 || rcStrict == VINF_VMX_VMEXIT
74 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76 || rcStrict == VINF_SVM_VMEXIT
77 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81 if ( rcStrict == VINF_VMX_VMEXIT
82 && rcPassUp == VINF_SUCCESS)
83 rcStrict = VINF_SUCCESS;
84 else
85#endif
86#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87 if ( rcStrict == VINF_SVM_VMEXIT
88 && rcPassUp == VINF_SUCCESS)
89 rcStrict = VINF_SUCCESS;
90 else
91#endif
92 if (rcPassUp == VINF_SUCCESS)
93 pVCpu->iem.s.cRetInfStatuses++;
94 else if ( rcPassUp < VINF_EM_FIRST
95 || rcPassUp > VINF_EM_LAST
96 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97 {
98 Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99 pVCpu->iem.s.cRetPassUpStatus++;
100 rcStrict = rcPassUp;
101 }
102 else
103 {
104 Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105 pVCpu->iem.s.cRetInfStatuses++;
106 }
107 }
108 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109 pVCpu->iem.s.cRetAspectNotImplemented++;
110 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111 pVCpu->iem.s.cRetInstrNotImplemented++;
112 else
113 pVCpu->iem.s.cRetErrStatuses++;
114 }
115 else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116 {
117 pVCpu->iem.s.cRetPassUpStatus++;
118 rcStrict = pVCpu->iem.s.rcPassUp;
119 }
120
121 return rcStrict;
122}
123
124
125/**
126 * Sets the pass up status.
127 *
128 * @returns VINF_SUCCESS.
129 * @param pVCpu The cross context virtual CPU structure of the
130 * calling thread.
131 * @param rcPassUp The pass up status. Must be informational.
132 * VINF_SUCCESS is not allowed.
133 */
134DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
135{
136 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139 if (rcOldPassUp == VINF_SUCCESS)
140 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141 /* If both are EM scheduling codes, use EM priority rules. */
142 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144 {
145 if (rcPassUp < rcOldPassUp)
146 {
147 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149 }
150 else
151 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152 }
153 /* Override EM scheduling with specific status code. */
154 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155 {
156 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158 }
159 /* Don't override specific status code, first come first served. */
160 else
161 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162 return VINF_SUCCESS;
163}
164
165
166/**
167 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
168 *
169 * Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
170 * reject expand down data segments and conforming code segments.
171 *
172 * ASSUMES that the CPU is in 32-bit mode.
173 *
174 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
175 * @param pVCpu The cross context virtual CPU structure of the
176 * calling thread.
177 * @sa iemCalc32BitFlatIndicatorEsDs
178 */
179DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
180{
181 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
182 return ( ( pVCpu->cpum.GstCtx.es.Attr.u
183 | pVCpu->cpum.GstCtx.cs.Attr.u
184 | pVCpu->cpum.GstCtx.ss.Attr.u
185 | pVCpu->cpum.GstCtx.ds.Attr.u)
186 & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P))
187 == (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)
188 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
189 | (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
190 | (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
191 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
192 == 0
193 && ( pVCpu->cpum.GstCtx.es.u64Base
194 | pVCpu->cpum.GstCtx.cs.u64Base
195 | pVCpu->cpum.GstCtx.ss.u64Base
196 | pVCpu->cpum.GstCtx.ds.u64Base)
197 == 0
198 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))
199 ? IEM_F_MODE_X86_32BIT_FLAT : 0;
200}
201
202
203/**
204 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
205 * flat already.
206 *
207 * This is used by sysenter.
208 *
209 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
210 * @param pVCpu The cross context virtual CPU structure of the
211 * calling thread.
212 * @sa iemCalc32BitFlatIndicator
213 */
214DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
215{
216 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
217 return ( ( pVCpu->cpum.GstCtx.es.Attr.u
218 | pVCpu->cpum.GstCtx.ds.Attr.u)
219 & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P))
220 == (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)
221 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
222 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
223 == 0
224 && ( pVCpu->cpum.GstCtx.es.u64Base
225 | pVCpu->cpum.GstCtx.ds.u64Base)
226 == 0
227 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))
228 ? IEM_F_MODE_X86_32BIT_FLAT : 0;
229}
230
231
232/**
233 * Calculates the IEM_F_MODE_XXX and CPL flags.
234 *
235 * @returns IEM_F_MODE_XXX
236 * @param pVCpu The cross context virtual CPU structure of the
237 * calling thread.
238 */
239DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
240{
241 /*
242 * We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
243 * here to try get this done as efficiently as possible.
244 */
245 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS);
246
247 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
248 {
249 if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
250 {
251 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
252 uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
253 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
254 {
255 Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
256 fExec |= IEM_F_MODE_X86_32BIT_PROT | iemCalc32BitFlatIndicator(pVCpu);
257 }
258 else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
259 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
260 fExec |= IEM_F_MODE_X86_64BIT;
261 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
262 fExec |= IEM_F_MODE_X86_16BIT_PROT;
263 else
264 fExec |= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
265 return fExec;
266 }
267 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
268 }
269
270 /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
271 if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
272 {
273 if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
274 return IEM_F_MODE_X86_16BIT;
275 return IEM_F_MODE_X86_16BIT_PRE_386;
276 }
277
278 /* 32-bit unreal mode. */
279 return IEM_F_MODE_X86_32BIT | iemCalc32BitFlatIndicator(pVCpu);
280}
281
282
283/**
284 * Calculates the AMD-V and VT-x related context flags.
285 *
286 * @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
287 * IEM_F_X86_CTX_VMX.
288 * @param pVCpu The cross context virtual CPU structure of the
289 * calling thread.
290 */
291DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
292{
293 /*
294 * This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
295 * and CPUMIsGuestInNestedHwvirtMode to some extent.
296 */
297 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
298
299 AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
300 uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
301 | (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
302 if (RT_LIKELY(!fTmp))
303 return 0; /* likely */
304
305 if (fTmp & X86_CR4_VMXE)
306 {
307 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
308 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
309 return IEM_F_X86_CTX_VMX | IEM_F_X86_CTX_IN_GUEST;
310 return IEM_F_X86_CTX_VMX;
311 }
312
313 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
314 if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
315 return IEM_F_X86_CTX_SVM | IEM_F_X86_CTX_IN_GUEST;
316 return IEM_F_X86_CTX_SVM;
317}
318
319
320/**
321 * Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
322 *
323 * @returns IEM_F_BRK_PENDING_XXX or zero.
324 * @param pVCpu The cross context virtual CPU structure of the
325 * calling thread.
326 */
327DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
328{
329 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
330
331 if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
332 && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
333 return 0;
334 return iemCalcExecDbgFlagsSlow(pVCpu);
335}
336
337/**
338 * Calculates the the IEM_F_XXX flags.
339 *
340 * @returns IEM_F_XXX combination match the current CPU state.
341 * @param pVCpu The cross context virtual CPU structure of the
342 * calling thread.
343 */
344DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
345{
346 return iemCalcExecModeAndCplFlags(pVCpu)
347 | iemCalcExecHwVirtFlags(pVCpu)
348 /* SMM is not yet implemented */
349 | iemCalcExecDbgFlags(pVCpu)
350 ;
351}
352
353
354/**
355 * Re-calculates the MODE and CPL parts of IEMCPU::fExec.
356 *
357 * @param pVCpu The cross context virtual CPU structure of the
358 * calling thread.
359 */
360DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
361{
362 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK | IEM_F_X86_CPL_MASK))
363 | iemCalcExecModeAndCplFlags(pVCpu);
364}
365
366
367/**
368 * Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
369 *
370 * @param pVCpu The cross context virtual CPU structure of the
371 * calling thread.
372 */
373DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
374{
375 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
376 | iemCalcExecDbgFlags(pVCpu);
377}
378
379
380#ifndef IEM_WITH_OPAQUE_DECODER_STATE
381
382# if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
383/**
384 * Initializes the execution state.
385 *
386 * @param pVCpu The cross context virtual CPU structure of the
387 * calling thread.
388 * @param fExecOpts Optional execution flags:
389 * - IEM_F_BYPASS_HANDLERS
390 * - IEM_F_X86_DISREGARD_LOCK
391 *
392 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
393 * side-effects in strict builds.
394 */
395DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
396{
397 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
398 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
399 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
400 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
401 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
402 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
403 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
404 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
405 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
406 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
407
408 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
409 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | fExecOpts;
410 pVCpu->iem.s.cActiveMappings = 0;
411 pVCpu->iem.s.iNextMapping = 0;
412
413# ifdef VBOX_STRICT
414 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
415 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
416 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
417 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
418 pVCpu->iem.s.fPrefixes = 0xfeedbeef;
419 pVCpu->iem.s.uRexReg = 127;
420 pVCpu->iem.s.uRexB = 127;
421 pVCpu->iem.s.offModRm = 127;
422 pVCpu->iem.s.uRexIndex = 127;
423 pVCpu->iem.s.iEffSeg = 127;
424 pVCpu->iem.s.idxPrefix = 127;
425 pVCpu->iem.s.uVex3rdReg = 127;
426 pVCpu->iem.s.uVexLength = 127;
427 pVCpu->iem.s.fEvexStuff = 127;
428 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
429# ifdef IEM_WITH_CODE_TLB
430 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
431 pVCpu->iem.s.pbInstrBuf = NULL;
432 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
433 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
434 pVCpu->iem.s.offCurInstrStart = INT16_MAX;
435 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
436# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
437 pVCpu->iem.s.offOpcode = 127;
438# endif
439# else
440 pVCpu->iem.s.offOpcode = 127;
441 pVCpu->iem.s.cbOpcode = 127;
442# endif
443# endif /* VBOX_STRICT */
444}
445# endif /* VBOX_INCLUDED_vmm_dbgf_h */
446
447
448# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
449/**
450 * Performs a minimal reinitialization of the execution state.
451 *
452 * This is intended to be used by VM-exits, SMM, LOADALL and other similar
453 * 'world-switch' types operations on the CPU. Currently only nested
454 * hardware-virtualization uses it.
455 *
456 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
457 * @param cbInstr The instruction length (for flushing).
458 */
459DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
460{
461 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
462 iemOpcodeFlushHeavy(pVCpu, cbInstr);
463}
464# endif
465
466
467/**
468 * Counterpart to #iemInitExec that undoes evil strict-build stuff.
469 *
470 * @param pVCpu The cross context virtual CPU structure of the
471 * calling thread.
472 */
473DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
474{
475 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
476# ifdef VBOX_STRICT
477# ifdef IEM_WITH_CODE_TLB
478 NOREF(pVCpu);
479# else
480 pVCpu->iem.s.cbOpcode = 0;
481# endif
482# else
483 NOREF(pVCpu);
484# endif
485}
486
487
488/**
489 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
490 *
491 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
492 *
493 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
494 * @param pVCpu The cross context virtual CPU structure of the calling thread.
495 * @param rcStrict The status code to fiddle.
496 */
497DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
498{
499 iemUninitExec(pVCpu);
500 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
501}
502
503
504/**
505 * Macro used by the IEMExec* method to check the given instruction length.
506 *
507 * Will return on failure!
508 *
509 * @param a_cbInstr The given instruction length.
510 * @param a_cbMin The minimum length.
511 */
512# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
513 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
514 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
515
516
517# ifndef IEM_WITH_SETJMP
518
519/**
520 * Fetches the first opcode byte.
521 *
522 * @returns Strict VBox status code.
523 * @param pVCpu The cross context virtual CPU structure of the
524 * calling thread.
525 * @param pu8 Where to return the opcode byte.
526 */
527DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
528{
529 /*
530 * Check for hardware instruction breakpoints.
531 */
532 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
533 { /* likely */ }
534 else
535 {
536 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
537 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
538 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
539 { /* likely */ }
540 else if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
541 return iemRaiseDebugException(pVCpu);
542 else
543 return rcStrict;
544 }
545
546 /*
547 * Fetch the first opcode byte.
548 */
549 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
550 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
551 {
552 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
553 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
554 return VINF_SUCCESS;
555 }
556 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
557}
558
559# else /* IEM_WITH_SETJMP */
560
561/**
562 * Fetches the first opcode byte, longjmp on error.
563 *
564 * @returns The opcode byte.
565 * @param pVCpu The cross context virtual CPU structure of the calling thread.
566 */
567DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
568{
569 /*
570 * Check for hardware instruction breakpoints.
571 */
572 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
573 { /* likely */ }
574 else
575 {
576 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
577 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
578 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
579 { /* likely */ }
580 else
581 {
582 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
583 rcStrict = iemRaiseDebugException(pVCpu);
584 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
585 }
586 }
587
588 /*
589 * Fetch the first opcode byte.
590 */
591# ifdef IEM_WITH_CODE_TLB
592 uint8_t bRet;
593 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
594 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
595 if (RT_LIKELY( pbBuf != NULL
596 && offBuf < pVCpu->iem.s.cbInstrBuf))
597 {
598 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
599 bRet = pbBuf[offBuf];
600 }
601 else
602 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
603# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
604 Assert(pVCpu->iem.s.offOpcode == 0);
605 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
606# endif
607 return bRet;
608
609# else /* !IEM_WITH_CODE_TLB */
610 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
611 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
612 {
613 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
614 return pVCpu->iem.s.abOpcode[offOpcode];
615 }
616 return iemOpcodeGetNextU8SlowJmp(pVCpu);
617# endif
618}
619
620# endif /* IEM_WITH_SETJMP */
621
622/**
623 * Fetches the first opcode byte, returns/throws automatically on failure.
624 *
625 * @param a_pu8 Where to return the opcode byte.
626 * @remark Implicitly references pVCpu.
627 */
628# ifndef IEM_WITH_SETJMP
629# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
630 do \
631 { \
632 VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
633 if (rcStrict2 == VINF_SUCCESS) \
634 { /* likely */ } \
635 else \
636 return rcStrict2; \
637 } while (0)
638# else
639# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
640# endif /* IEM_WITH_SETJMP */
641
642
643# ifndef IEM_WITH_SETJMP
644
645/**
646 * Fetches the next opcode byte.
647 *
648 * @returns Strict VBox status code.
649 * @param pVCpu The cross context virtual CPU structure of the
650 * calling thread.
651 * @param pu8 Where to return the opcode byte.
652 */
653DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
654{
655 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
656 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
657 {
658 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
659 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
660 return VINF_SUCCESS;
661 }
662 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
663}
664
665# else /* IEM_WITH_SETJMP */
666
667/**
668 * Fetches the next opcode byte, longjmp on error.
669 *
670 * @returns The opcode byte.
671 * @param pVCpu The cross context virtual CPU structure of the calling thread.
672 */
673DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
674{
675# ifdef IEM_WITH_CODE_TLB
676 uint8_t bRet;
677 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
678 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
679 if (RT_LIKELY( pbBuf != NULL
680 && offBuf < pVCpu->iem.s.cbInstrBuf))
681 {
682 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
683 bRet = pbBuf[offBuf];
684 }
685 else
686 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
687# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
688 Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
689 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
690# endif
691 return bRet;
692
693# else /* !IEM_WITH_CODE_TLB */
694 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
695 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
696 {
697 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
698 return pVCpu->iem.s.abOpcode[offOpcode];
699 }
700 return iemOpcodeGetNextU8SlowJmp(pVCpu);
701# endif
702}
703
704# endif /* IEM_WITH_SETJMP */
705
706/**
707 * Fetches the next opcode byte, returns automatically on failure.
708 *
709 * @param a_pu8 Where to return the opcode byte.
710 * @remark Implicitly references pVCpu.
711 */
712# ifndef IEM_WITH_SETJMP
713# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
714 do \
715 { \
716 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
717 if (rcStrict2 == VINF_SUCCESS) \
718 { /* likely */ } \
719 else \
720 return rcStrict2; \
721 } while (0)
722# else
723# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
724# endif /* IEM_WITH_SETJMP */
725
726
727# ifndef IEM_WITH_SETJMP
728/**
729 * Fetches the next signed byte from the opcode stream.
730 *
731 * @returns Strict VBox status code.
732 * @param pVCpu The cross context virtual CPU structure of the calling thread.
733 * @param pi8 Where to return the signed byte.
734 */
735DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
736{
737 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
738}
739# endif /* !IEM_WITH_SETJMP */
740
741
742/**
743 * Fetches the next signed byte from the opcode stream, returning automatically
744 * on failure.
745 *
746 * @param a_pi8 Where to return the signed byte.
747 * @remark Implicitly references pVCpu.
748 */
749# ifndef IEM_WITH_SETJMP
750# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
751 do \
752 { \
753 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
754 if (rcStrict2 != VINF_SUCCESS) \
755 return rcStrict2; \
756 } while (0)
757# else /* IEM_WITH_SETJMP */
758# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
759
760# endif /* IEM_WITH_SETJMP */
761
762
763# ifndef IEM_WITH_SETJMP
764/**
765 * Fetches the next signed byte from the opcode stream, extending it to
766 * unsigned 16-bit.
767 *
768 * @returns Strict VBox status code.
769 * @param pVCpu The cross context virtual CPU structure of the calling thread.
770 * @param pu16 Where to return the unsigned word.
771 */
772DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
773{
774 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
775 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
776 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
777
778 *pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
779 pVCpu->iem.s.offOpcode = offOpcode + 1;
780 return VINF_SUCCESS;
781}
782# endif /* !IEM_WITH_SETJMP */
783
784/**
785 * Fetches the next signed byte from the opcode stream and sign-extending it to
786 * a word, returning automatically on failure.
787 *
788 * @param a_pu16 Where to return the word.
789 * @remark Implicitly references pVCpu.
790 */
791# ifndef IEM_WITH_SETJMP
792# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
793 do \
794 { \
795 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
796 if (rcStrict2 != VINF_SUCCESS) \
797 return rcStrict2; \
798 } while (0)
799# else
800# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
801# endif
802
803# ifndef IEM_WITH_SETJMP
804/**
805 * Fetches the next signed byte from the opcode stream, extending it to
806 * unsigned 32-bit.
807 *
808 * @returns Strict VBox status code.
809 * @param pVCpu The cross context virtual CPU structure of the calling thread.
810 * @param pu32 Where to return the unsigned dword.
811 */
812DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
813{
814 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
815 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
816 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
817
818 *pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
819 pVCpu->iem.s.offOpcode = offOpcode + 1;
820 return VINF_SUCCESS;
821}
822# endif /* !IEM_WITH_SETJMP */
823
824/**
825 * Fetches the next signed byte from the opcode stream and sign-extending it to
826 * a word, returning automatically on failure.
827 *
828 * @param a_pu32 Where to return the word.
829 * @remark Implicitly references pVCpu.
830 */
831# ifndef IEM_WITH_SETJMP
832# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
833 do \
834 { \
835 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
836 if (rcStrict2 != VINF_SUCCESS) \
837 return rcStrict2; \
838 } while (0)
839# else
840# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
841# endif
842
843
844# ifndef IEM_WITH_SETJMP
845/**
846 * Fetches the next signed byte from the opcode stream, extending it to
847 * unsigned 64-bit.
848 *
849 * @returns Strict VBox status code.
850 * @param pVCpu The cross context virtual CPU structure of the calling thread.
851 * @param pu64 Where to return the unsigned qword.
852 */
853DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
854{
855 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
856 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
857 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
858
859 *pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
860 pVCpu->iem.s.offOpcode = offOpcode + 1;
861 return VINF_SUCCESS;
862}
863# endif /* !IEM_WITH_SETJMP */
864
865/**
866 * Fetches the next signed byte from the opcode stream and sign-extending it to
867 * a word, returning automatically on failure.
868 *
869 * @param a_pu64 Where to return the word.
870 * @remark Implicitly references pVCpu.
871 */
872# ifndef IEM_WITH_SETJMP
873# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
874 do \
875 { \
876 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
877 if (rcStrict2 != VINF_SUCCESS) \
878 return rcStrict2; \
879 } while (0)
880# else
881# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
882# endif
883
884
885# ifndef IEM_WITH_SETJMP
886
887/**
888 * Fetches the next opcode word.
889 *
890 * @returns Strict VBox status code.
891 * @param pVCpu The cross context virtual CPU structure of the calling thread.
892 * @param pu16 Where to return the opcode word.
893 */
894DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
895{
896 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
897 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
898 {
899 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
900# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
901 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
902# else
903 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
904# endif
905 return VINF_SUCCESS;
906 }
907 return iemOpcodeGetNextU16Slow(pVCpu, pu16);
908}
909
910# else /* IEM_WITH_SETJMP */
911
912/**
913 * Fetches the next opcode word, longjmp on error.
914 *
915 * @returns The opcode word.
916 * @param pVCpu The cross context virtual CPU structure of the calling thread.
917 */
918DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
919{
920# ifdef IEM_WITH_CODE_TLB
921 uint16_t u16Ret;
922 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
923 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
924 if (RT_LIKELY( pbBuf != NULL
925 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
926 {
927 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
928# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
929 u16Ret = *(uint16_t const *)&pbBuf[offBuf];
930# else
931 u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
932# endif
933 }
934 else
935 u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
936
937# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
938 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
939 Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
940# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
941 *(uint16_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
942# else
943 pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
944 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
945# endif
946 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
947# endif
948
949 return u16Ret;
950
951# else /* !IEM_WITH_CODE_TLB */
952 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
953 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
954 {
955 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
956# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
957 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
958# else
959 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
960# endif
961 }
962 return iemOpcodeGetNextU16SlowJmp(pVCpu);
963# endif /* !IEM_WITH_CODE_TLB */
964}
965
966# endif /* IEM_WITH_SETJMP */
967
968/**
969 * Fetches the next opcode word, returns automatically on failure.
970 *
971 * @param a_pu16 Where to return the opcode word.
972 * @remark Implicitly references pVCpu.
973 */
974# ifndef IEM_WITH_SETJMP
975# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
976 do \
977 { \
978 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
979 if (rcStrict2 != VINF_SUCCESS) \
980 return rcStrict2; \
981 } while (0)
982# else
983# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
984# endif
985
986# ifndef IEM_WITH_SETJMP
987/**
988 * Fetches the next opcode word, zero extending it to a double word.
989 *
990 * @returns Strict VBox status code.
991 * @param pVCpu The cross context virtual CPU structure of the calling thread.
992 * @param pu32 Where to return the opcode double word.
993 */
994DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
995{
996 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
997 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
998 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
999
1000 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1001 pVCpu->iem.s.offOpcode = offOpcode + 2;
1002 return VINF_SUCCESS;
1003}
1004# endif /* !IEM_WITH_SETJMP */
1005
1006/**
1007 * Fetches the next opcode word and zero extends it to a double word, returns
1008 * automatically on failure.
1009 *
1010 * @param a_pu32 Where to return the opcode double word.
1011 * @remark Implicitly references pVCpu.
1012 */
1013# ifndef IEM_WITH_SETJMP
1014# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1015 do \
1016 { \
1017 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1018 if (rcStrict2 != VINF_SUCCESS) \
1019 return rcStrict2; \
1020 } while (0)
1021# else
1022# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1023# endif
1024
1025# ifndef IEM_WITH_SETJMP
1026/**
1027 * Fetches the next opcode word, zero extending it to a quad word.
1028 *
1029 * @returns Strict VBox status code.
1030 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1031 * @param pu64 Where to return the opcode quad word.
1032 */
1033DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1034{
1035 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1036 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1037 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1038
1039 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1040 pVCpu->iem.s.offOpcode = offOpcode + 2;
1041 return VINF_SUCCESS;
1042}
1043# endif /* !IEM_WITH_SETJMP */
1044
1045/**
1046 * Fetches the next opcode word and zero extends it to a quad word, returns
1047 * automatically on failure.
1048 *
1049 * @param a_pu64 Where to return the opcode quad word.
1050 * @remark Implicitly references pVCpu.
1051 */
1052# ifndef IEM_WITH_SETJMP
1053# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1054 do \
1055 { \
1056 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1057 if (rcStrict2 != VINF_SUCCESS) \
1058 return rcStrict2; \
1059 } while (0)
1060# else
1061# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1062# endif
1063
1064
1065# ifndef IEM_WITH_SETJMP
1066/**
1067 * Fetches the next signed word from the opcode stream.
1068 *
1069 * @returns Strict VBox status code.
1070 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1071 * @param pi16 Where to return the signed word.
1072 */
1073DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1074{
1075 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1076}
1077# endif /* !IEM_WITH_SETJMP */
1078
1079
1080/**
1081 * Fetches the next signed word from the opcode stream, returning automatically
1082 * on failure.
1083 *
1084 * @param a_pi16 Where to return the signed word.
1085 * @remark Implicitly references pVCpu.
1086 */
1087# ifndef IEM_WITH_SETJMP
1088# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1089 do \
1090 { \
1091 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1092 if (rcStrict2 != VINF_SUCCESS) \
1093 return rcStrict2; \
1094 } while (0)
1095# else
1096# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1097# endif
1098
1099# ifndef IEM_WITH_SETJMP
1100
1101/**
1102 * Fetches the next opcode dword.
1103 *
1104 * @returns Strict VBox status code.
1105 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1106 * @param pu32 Where to return the opcode double word.
1107 */
1108DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1109{
1110 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1111 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1112 {
1113 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1114# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1115 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1116# else
1117 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1118 pVCpu->iem.s.abOpcode[offOpcode + 1],
1119 pVCpu->iem.s.abOpcode[offOpcode + 2],
1120 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1121# endif
1122 return VINF_SUCCESS;
1123 }
1124 return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1125}
1126
1127# else /* IEM_WITH_SETJMP */
1128
1129/**
1130 * Fetches the next opcode dword, longjmp on error.
1131 *
1132 * @returns The opcode dword.
1133 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1134 */
1135DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1136{
1137# ifdef IEM_WITH_CODE_TLB
1138 uint32_t u32Ret;
1139 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1140 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1141 if (RT_LIKELY( pbBuf != NULL
1142 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1143 {
1144 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1145# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1146 u32Ret = *(uint32_t const *)&pbBuf[offBuf];
1147# else
1148 u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1149 pbBuf[offBuf + 1],
1150 pbBuf[offBuf + 2],
1151 pbBuf[offBuf + 3]);
1152# endif
1153 }
1154 else
1155 u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1156
1157# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1158 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1159 Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1160# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1161 *(uint32_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1162# else
1163 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1164 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1165 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1166 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1167# endif
1168 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1169# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1170
1171 return u32Ret;
1172
1173# else /* !IEM_WITH_CODE_TLB */
1174 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1175 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1176 {
1177 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1178# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1179 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1180# else
1181 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1182 pVCpu->iem.s.abOpcode[offOpcode + 1],
1183 pVCpu->iem.s.abOpcode[offOpcode + 2],
1184 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1185# endif
1186 }
1187 return iemOpcodeGetNextU32SlowJmp(pVCpu);
1188# endif
1189}
1190
1191# endif /* IEM_WITH_SETJMP */
1192
1193/**
1194 * Fetches the next opcode dword, returns automatically on failure.
1195 *
1196 * @param a_pu32 Where to return the opcode dword.
1197 * @remark Implicitly references pVCpu.
1198 */
1199# ifndef IEM_WITH_SETJMP
1200# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1201 do \
1202 { \
1203 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1204 if (rcStrict2 != VINF_SUCCESS) \
1205 return rcStrict2; \
1206 } while (0)
1207# else
1208# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1209# endif
1210
1211# ifndef IEM_WITH_SETJMP
1212/**
1213 * Fetches the next opcode dword, zero extending it to a quad word.
1214 *
1215 * @returns Strict VBox status code.
1216 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1217 * @param pu64 Where to return the opcode quad word.
1218 */
1219DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1220{
1221 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1222 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1223 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1224
1225 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1226 pVCpu->iem.s.abOpcode[offOpcode + 1],
1227 pVCpu->iem.s.abOpcode[offOpcode + 2],
1228 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1229 pVCpu->iem.s.offOpcode = offOpcode + 4;
1230 return VINF_SUCCESS;
1231}
1232# endif /* !IEM_WITH_SETJMP */
1233
1234/**
1235 * Fetches the next opcode dword and zero extends it to a quad word, returns
1236 * automatically on failure.
1237 *
1238 * @param a_pu64 Where to return the opcode quad word.
1239 * @remark Implicitly references pVCpu.
1240 */
1241# ifndef IEM_WITH_SETJMP
1242# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1243 do \
1244 { \
1245 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1246 if (rcStrict2 != VINF_SUCCESS) \
1247 return rcStrict2; \
1248 } while (0)
1249# else
1250# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1251# endif
1252
1253
1254# ifndef IEM_WITH_SETJMP
1255/**
1256 * Fetches the next signed double word from the opcode stream.
1257 *
1258 * @returns Strict VBox status code.
1259 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1260 * @param pi32 Where to return the signed double word.
1261 */
1262DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1263{
1264 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1265}
1266# endif
1267
1268/**
1269 * Fetches the next signed double word from the opcode stream, returning
1270 * automatically on failure.
1271 *
1272 * @param a_pi32 Where to return the signed double word.
1273 * @remark Implicitly references pVCpu.
1274 */
1275# ifndef IEM_WITH_SETJMP
1276# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1277 do \
1278 { \
1279 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1280 if (rcStrict2 != VINF_SUCCESS) \
1281 return rcStrict2; \
1282 } while (0)
1283# else
1284# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1285# endif
1286
1287# ifndef IEM_WITH_SETJMP
1288/**
1289 * Fetches the next opcode dword, sign extending it into a quad word.
1290 *
1291 * @returns Strict VBox status code.
1292 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1293 * @param pu64 Where to return the opcode quad word.
1294 */
1295DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1296{
1297 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1298 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1299 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1300
1301 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1302 pVCpu->iem.s.abOpcode[offOpcode + 1],
1303 pVCpu->iem.s.abOpcode[offOpcode + 2],
1304 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1305 *pu64 = (uint64_t)(int64_t)i32;
1306 pVCpu->iem.s.offOpcode = offOpcode + 4;
1307 return VINF_SUCCESS;
1308}
1309# endif /* !IEM_WITH_SETJMP */
1310
1311/**
1312 * Fetches the next opcode double word and sign extends it to a quad word,
1313 * returns automatically on failure.
1314 *
1315 * @param a_pu64 Where to return the opcode quad word.
1316 * @remark Implicitly references pVCpu.
1317 */
1318# ifndef IEM_WITH_SETJMP
1319# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1320 do \
1321 { \
1322 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1323 if (rcStrict2 != VINF_SUCCESS) \
1324 return rcStrict2; \
1325 } while (0)
1326# else
1327# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1328# endif
1329
1330# ifndef IEM_WITH_SETJMP
1331
1332/**
1333 * Fetches the next opcode qword.
1334 *
1335 * @returns Strict VBox status code.
1336 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1337 * @param pu64 Where to return the opcode qword.
1338 */
1339DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1340{
1341 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1342 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1343 {
1344# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1345 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1346# else
1347 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1348 pVCpu->iem.s.abOpcode[offOpcode + 1],
1349 pVCpu->iem.s.abOpcode[offOpcode + 2],
1350 pVCpu->iem.s.abOpcode[offOpcode + 3],
1351 pVCpu->iem.s.abOpcode[offOpcode + 4],
1352 pVCpu->iem.s.abOpcode[offOpcode + 5],
1353 pVCpu->iem.s.abOpcode[offOpcode + 6],
1354 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1355# endif
1356 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1357 return VINF_SUCCESS;
1358 }
1359 return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1360}
1361
1362# else /* IEM_WITH_SETJMP */
1363
1364/**
1365 * Fetches the next opcode qword, longjmp on error.
1366 *
1367 * @returns The opcode qword.
1368 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1369 */
1370DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1371{
1372# ifdef IEM_WITH_CODE_TLB
1373 uint64_t u64Ret;
1374 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1375 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1376 if (RT_LIKELY( pbBuf != NULL
1377 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1378 {
1379 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1380# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1381 u64Ret = *(uint64_t const *)&pbBuf[offBuf];
1382# else
1383 u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1384 pbBuf[offBuf + 1],
1385 pbBuf[offBuf + 2],
1386 pbBuf[offBuf + 3],
1387 pbBuf[offBuf + 4],
1388 pbBuf[offBuf + 5],
1389 pbBuf[offBuf + 6],
1390 pbBuf[offBuf + 7]);
1391# endif
1392 }
1393 else
1394 u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1395
1396# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1397 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1398 Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1399# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1400 *(uint64_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1401# else
1402 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1403 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1404 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1405 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1406 pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1407 pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1408 pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1409 pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1410# endif
1411 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1412# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1413
1414 return u64Ret;
1415
1416# else /* !IEM_WITH_CODE_TLB */
1417 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1418 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1419 {
1420 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1421# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1422 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1423# else
1424 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1425 pVCpu->iem.s.abOpcode[offOpcode + 1],
1426 pVCpu->iem.s.abOpcode[offOpcode + 2],
1427 pVCpu->iem.s.abOpcode[offOpcode + 3],
1428 pVCpu->iem.s.abOpcode[offOpcode + 4],
1429 pVCpu->iem.s.abOpcode[offOpcode + 5],
1430 pVCpu->iem.s.abOpcode[offOpcode + 6],
1431 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1432# endif
1433 }
1434 return iemOpcodeGetNextU64SlowJmp(pVCpu);
1435# endif /* !IEM_WITH_CODE_TLB */
1436}
1437
1438# endif /* IEM_WITH_SETJMP */
1439
1440/**
1441 * Fetches the next opcode quad word, returns automatically on failure.
1442 *
1443 * @param a_pu64 Where to return the opcode quad word.
1444 * @remark Implicitly references pVCpu.
1445 */
1446# ifndef IEM_WITH_SETJMP
1447# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1448 do \
1449 { \
1450 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1451 if (rcStrict2 != VINF_SUCCESS) \
1452 return rcStrict2; \
1453 } while (0)
1454# else
1455# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1456# endif
1457
1458/**
1459 * For fetching the opcode bytes for an ModR/M effective address, but throw
1460 * away the result.
1461 *
1462 * This is used when decoding undefined opcodes and such where we want to avoid
1463 * unnecessary MC blocks.
1464 *
1465 * @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1466 * used instead. At least for now...
1467 */
1468# ifndef IEM_WITH_SETJMP
1469# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1470 RTGCPTR GCPtrEff; \
1471 VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1472 if (rcStrict != VINF_SUCCESS) \
1473 return rcStrict; \
1474 } while (0)
1475# else
1476# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1477 (void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1478 } while (0)
1479# endif
1480
1481#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1482
1483
1484/** @name Misc Worker Functions.
1485 * @{
1486 */
1487
1488/**
1489 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1490 * not (kind of obsolete now).
1491 *
1492 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1493 */
1494#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1495
1496/**
1497 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1498 *
1499 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1500 * @param a_fEfl The new EFLAGS.
1501 */
1502#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1503
1504
1505/**
1506 * Loads a NULL data selector into a selector register, both the hidden and
1507 * visible parts, in protected mode.
1508 *
1509 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1510 * @param pSReg Pointer to the segment register.
1511 * @param uRpl The RPL.
1512 */
1513DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1514{
1515 /** @todo Testcase: write a testcase checking what happends when loading a NULL
1516 * data selector in protected mode. */
1517 pSReg->Sel = uRpl;
1518 pSReg->ValidSel = uRpl;
1519 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1520 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1521 {
1522 /* VT-x (Intel 3960x) observed doing something like this. */
1523 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1524 pSReg->u32Limit = UINT32_MAX;
1525 pSReg->u64Base = 0;
1526 }
1527 else
1528 {
1529 pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1530 pSReg->u32Limit = 0;
1531 pSReg->u64Base = 0;
1532 }
1533}
1534
1535/** @} */
1536
1537
1538/*
1539 *
1540 * Helpers routines.
1541 * Helpers routines.
1542 * Helpers routines.
1543 *
1544 */
1545
1546#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1547
1548/**
1549 * Recalculates the effective operand size.
1550 *
1551 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1552 */
1553DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1554{
1555 switch (IEM_GET_CPU_MODE(pVCpu))
1556 {
1557 case IEMMODE_16BIT:
1558 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1559 break;
1560 case IEMMODE_32BIT:
1561 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1562 break;
1563 case IEMMODE_64BIT:
1564 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
1565 {
1566 case 0:
1567 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1568 break;
1569 case IEM_OP_PRF_SIZE_OP:
1570 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1571 break;
1572 case IEM_OP_PRF_SIZE_REX_W:
1573 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
1574 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1575 break;
1576 }
1577 break;
1578 default:
1579 AssertFailed();
1580 }
1581}
1582
1583
1584/**
1585 * Sets the default operand size to 64-bit and recalculates the effective
1586 * operand size.
1587 *
1588 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1589 */
1590DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1591{
1592 Assert(IEM_IS_64BIT_CODE(pVCpu));
1593 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1594 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1595 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1596 else
1597 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1598}
1599
1600
1601/**
1602 * Sets the default operand size to 64-bit and recalculates the effective
1603 * operand size, with intel ignoring any operand size prefix (AMD respects it).
1604 *
1605 * This is for the relative jumps.
1606 *
1607 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1608 */
1609DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1610{
1611 Assert(IEM_IS_64BIT_CODE(pVCpu));
1612 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1613 if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1614 || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1615 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1616 else
1617 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1618}
1619
1620#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1621
1622
1623
1624/** @name Register Access.
1625 * @{
1626 */
1627
1628/**
1629 * Gets a reference (pointer) to the specified hidden segment register.
1630 *
1631 * @returns Hidden register reference.
1632 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1633 * @param iSegReg The segment register.
1634 */
1635DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1636{
1637 Assert(iSegReg < X86_SREG_COUNT);
1638 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1639 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1640
1641 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1642 return pSReg;
1643}
1644
1645
1646/**
1647 * Ensures that the given hidden segment register is up to date.
1648 *
1649 * @returns Hidden register reference.
1650 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1651 * @param pSReg The segment register.
1652 */
1653DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1654{
1655 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1656 NOREF(pVCpu);
1657 return pSReg;
1658}
1659
1660
1661/**
1662 * Gets a reference (pointer) to the specified segment register (the selector
1663 * value).
1664 *
1665 * @returns Pointer to the selector variable.
1666 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1667 * @param iSegReg The segment register.
1668 */
1669DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1670{
1671 Assert(iSegReg < X86_SREG_COUNT);
1672 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1673 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1674}
1675
1676
1677/**
1678 * Fetches the selector value of a segment register.
1679 *
1680 * @returns The selector value.
1681 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1682 * @param iSegReg The segment register.
1683 */
1684DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1685{
1686 Assert(iSegReg < X86_SREG_COUNT);
1687 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1688 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1689}
1690
1691
1692/**
1693 * Fetches the base address value of a segment register.
1694 *
1695 * @returns The selector value.
1696 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1697 * @param iSegReg The segment register.
1698 */
1699DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1700{
1701 Assert(iSegReg < X86_SREG_COUNT);
1702 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1703 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1704}
1705
1706
1707/**
1708 * Gets a reference (pointer) to the specified general purpose register.
1709 *
1710 * @returns Register reference.
1711 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1712 * @param iReg The general purpose register.
1713 */
1714DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1715{
1716 Assert(iReg < 16);
1717 return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1718}
1719
1720
1721#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1722/**
1723 * Gets a reference (pointer) to the specified 8-bit general purpose register.
1724 *
1725 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1726 *
1727 * @returns Register reference.
1728 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1729 * @param iReg The register.
1730 */
1731DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1732{
1733 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1734 {
1735 Assert(iReg < 16);
1736 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1737 }
1738 /* high 8-bit register. */
1739 Assert(iReg < 8);
1740 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1741}
1742#endif
1743
1744
1745/**
1746 * Gets a reference (pointer) to the specified 8-bit general purpose register,
1747 * alternative version with extended (20) register index.
1748 *
1749 * @returns Register reference.
1750 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1751 * @param iRegEx The register. The 16 first are regular ones,
1752 * whereas 16 thru 19 maps to AH, CH, DH and BH.
1753 */
1754DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1755{
1756 /** @todo This could be done by double indexing on little endian hosts:
1757 * return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1758 if (iRegEx < 16)
1759 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1760
1761 /* high 8-bit register. */
1762 Assert(iRegEx < 20);
1763 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1764}
1765
1766
1767/**
1768 * Gets a reference (pointer) to the specified 16-bit general purpose register.
1769 *
1770 * @returns Register reference.
1771 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1772 * @param iReg The register.
1773 */
1774DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1775{
1776 Assert(iReg < 16);
1777 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1778}
1779
1780
1781/**
1782 * Gets a reference (pointer) to the specified 32-bit general purpose register.
1783 *
1784 * @returns Register reference.
1785 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1786 * @param iReg The register.
1787 */
1788DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1789{
1790 Assert(iReg < 16);
1791 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1792}
1793
1794
1795/**
1796 * Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1797 *
1798 * @returns Register reference.
1799 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1800 * @param iReg The register.
1801 */
1802DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1803{
1804 Assert(iReg < 16);
1805 return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1806}
1807
1808
1809/**
1810 * Gets a reference (pointer) to the specified 64-bit general purpose register.
1811 *
1812 * @returns Register reference.
1813 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1814 * @param iReg The register.
1815 */
1816DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1817{
1818 Assert(iReg < 64);
1819 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1820}
1821
1822
1823/**
1824 * Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1825 *
1826 * @returns Register reference.
1827 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1828 * @param iReg The register.
1829 */
1830DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1831{
1832 Assert(iReg < 16);
1833 return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1834}
1835
1836
1837/**
1838 * Gets a reference (pointer) to the specified segment register's base address.
1839 *
1840 * @returns Segment register base address reference.
1841 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1842 * @param iSegReg The segment selector.
1843 */
1844DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1845{
1846 Assert(iSegReg < X86_SREG_COUNT);
1847 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1848 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1849}
1850
1851
1852#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1853/**
1854 * Fetches the value of a 8-bit general purpose register.
1855 *
1856 * @returns The register value.
1857 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1858 * @param iReg The register.
1859 */
1860DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1861{
1862 return *iemGRegRefU8(pVCpu, iReg);
1863}
1864#endif
1865
1866
1867/**
1868 * Fetches the value of a 8-bit general purpose register, alternative version
1869 * with extended (20) register index.
1870
1871 * @returns The register value.
1872 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1873 * @param iRegEx The register. The 16 first are regular ones,
1874 * whereas 16 thru 19 maps to AH, CH, DH and BH.
1875 */
1876DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1877{
1878 return *iemGRegRefU8Ex(pVCpu, iRegEx);
1879}
1880
1881
1882/**
1883 * Fetches the value of a 16-bit general purpose register.
1884 *
1885 * @returns The register value.
1886 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1887 * @param iReg The register.
1888 */
1889DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1890{
1891 Assert(iReg < 16);
1892 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1893}
1894
1895
1896/**
1897 * Fetches the value of a 32-bit general purpose register.
1898 *
1899 * @returns The register value.
1900 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1901 * @param iReg The register.
1902 */
1903DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1904{
1905 Assert(iReg < 16);
1906 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1907}
1908
1909
1910/**
1911 * Fetches the value of a 64-bit general purpose register.
1912 *
1913 * @returns The register value.
1914 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1915 * @param iReg The register.
1916 */
1917DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1918{
1919 Assert(iReg < 16);
1920 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1921}
1922
1923
1924/**
1925 * Stores a 16-bit value to a general purpose register.
1926 *
1927 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1928 * @param iReg The register.
1929 * @param uValue The value to store.
1930 */
1931DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1932{
1933 Assert(iReg < 16);
1934 pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1935}
1936
1937
1938/**
1939 * Stores a 32-bit value to a general purpose register, implicitly clearing high
1940 * values.
1941 *
1942 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1943 * @param iReg The register.
1944 * @param uValue The value to store.
1945 */
1946DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1947{
1948 Assert(iReg < 16);
1949 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1950}
1951
1952
1953/**
1954 * Stores a 64-bit value to a general purpose register.
1955 *
1956 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1957 * @param iReg The register.
1958 * @param uValue The value to store.
1959 */
1960DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1961{
1962 Assert(iReg < 16);
1963 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1964}
1965
1966
1967/**
1968 * Get the address of the top of the stack.
1969 *
1970 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1971 */
1972DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1973{
1974 if (IEM_IS_64BIT_CODE(pVCpu))
1975 return pVCpu->cpum.GstCtx.rsp;
1976 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1977 return pVCpu->cpum.GstCtx.esp;
1978 return pVCpu->cpum.GstCtx.sp;
1979}
1980
1981
1982/**
1983 * Updates the RIP/EIP/IP to point to the next instruction.
1984 *
1985 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1986 * @param cbInstr The number of bytes to add.
1987 */
1988DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1989{
1990 /*
1991 * Advance RIP.
1992 *
1993 * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1994 * while in all other modes except LM64 the updates are 32-bit. This means
1995 * we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1996 * 4GB and 64KB rollovers, and decide whether anything needs masking.
1997 *
1998 * See PC wrap around tests in bs3-cpu-weird-1.
1999 */
2000 uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2001 uint64_t const uRipNext = uRipPrev + cbInstr;
2002 if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16)))
2003 || IEM_IS_64BIT_CODE(pVCpu)))
2004 pVCpu->cpum.GstCtx.rip = uRipNext;
2005 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2006 pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2007 else
2008 pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2009}
2010
2011
2012/**
2013 * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2014 * following EFLAGS bits are set:
2015 * - X86_EFL_RF - clear it.
2016 * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2017 * - X86_EFL_TF - generate single step \#DB trap.
2018 * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2019 * instruction).
2020 *
2021 * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2022 * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2023 * takes priority over both NMIs and hardware interrupts. So, neither is
2024 * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2025 * either unsupported will be triggered on-top of any \#DB raised here.)
2026 *
2027 * The RF flag only needs to be cleared here as it only suppresses instruction
2028 * breakpoints which are not raised here (happens synchronously during
2029 * instruction fetching).
2030 *
2031 * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2032 * status has no bearing on whether \#DB exceptions are raised.
2033 *
2034 * @note This must *NOT* be called by the two instructions setting the
2035 * CPUMCTX_INHIBIT_SHADOW_SS flag.
2036 *
2037 * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2038 * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2039 * Stacks}
2040 */
2041static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2042{
2043 /*
2044 * Normally we're just here to clear RF and/or interrupt shadow bits.
2045 */
2046 if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0))
2047 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);
2048 else
2049 {
2050 /*
2051 * Raise a #DB or/and DBGF event.
2052 */
2053 VBOXSTRICTRC rcStrict;
2054 if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK))
2055 {
2056 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2057 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2058 if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2059 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS;
2060 pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2061 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2062 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2063 pVCpu->cpum.GstCtx.rflags.uBoth));
2064
2065 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK);
2066 rcStrict = iemRaiseDebugException(pVCpu);
2067
2068 /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2069 if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2070 {
2071 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2072 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2073 }
2074 }
2075 else
2076 {
2077 Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2078 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2079 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2080 }
2081 pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2082 return rcStrict;
2083 }
2084 return VINF_SUCCESS;
2085}
2086
2087
2088/**
2089 * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2090 *
2091 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2092 */
2093DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2094{
2095 /*
2096 * We assume that most of the time nothing actually needs doing here.
2097 */
2098 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2099 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2100 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))
2101 return VINF_SUCCESS;
2102 return iemFinishInstructionWithFlagsSet(pVCpu);
2103}
2104
2105
2106/**
2107 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2108 * and CPUMCTX_INHIBIT_SHADOW.
2109 *
2110 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2111 * @param cbInstr The number of bytes to add.
2112 */
2113DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2114{
2115 iemRegAddToRip(pVCpu, cbInstr);
2116 return iemRegFinishClearingRF(pVCpu);
2117}
2118
2119
2120/**
2121 * Updates the RIP to point to the next instruction and clears EFLAGS.RF
2122 * and CPUMCTX_INHIBIT_SHADOW.
2123 *
2124 * Only called from 64-bit code.
2125 *
2126 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2127 * @param cbInstr The number of bytes to add.
2128 */
2129DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2130{
2131 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2132 return iemRegFinishClearingRF(pVCpu);
2133}
2134
2135
2136/**
2137 * Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2138 * CPUMCTX_INHIBIT_SHADOW.
2139 *
2140 * This is never from 64-bit code.
2141 *
2142 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2143 * @param cbInstr The number of bytes to add.
2144 */
2145DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2146{
2147 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2148 return iemRegFinishClearingRF(pVCpu);
2149}
2150
2151
2152/**
2153 * Updates the IP to point to the next instruction and clears EFLAGS.RF and
2154 * CPUMCTX_INHIBIT_SHADOW.
2155 *
2156 * This is only ever used from 16-bit code on a pre-386 CPU.
2157 *
2158 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2159 * @param cbInstr The number of bytes to add.
2160 */
2161DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2162{
2163 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2164 return iemRegFinishClearingRF(pVCpu);
2165}
2166
2167
2168/**
2169 * Adds a 8-bit signed jump offset to RIP from 64-bit code.
2170 *
2171 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2172 * segment limit.
2173 *
2174 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2175 * @param cbInstr Instruction size.
2176 * @param offNextInstr The offset of the next instruction.
2177 * @param enmEffOpSize Effective operand size.
2178 */
2179DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2180 IEMMODE enmEffOpSize) RT_NOEXCEPT
2181{
2182 Assert(IEM_IS_64BIT_CODE(pVCpu));
2183 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);
2184
2185 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2186 if (enmEffOpSize == IEMMODE_16BIT)
2187 uNewRip &= UINT16_MAX;
2188
2189 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2190 pVCpu->cpum.GstCtx.rip = uNewRip;
2191 else
2192 return iemRaiseGeneralProtectionFault0(pVCpu);
2193
2194#ifndef IEM_WITH_CODE_TLB
2195 iemOpcodeFlushLight(pVCpu, cbInstr);
2196#endif
2197
2198 /*
2199 * Clear RF and finish the instruction (maybe raise #DB).
2200 */
2201 return iemRegFinishClearingRF(pVCpu);
2202}
2203
2204
2205/**
2206 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2207 * code (never 64-bit).
2208 *
2209 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2210 * segment limit.
2211 *
2212 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2213 * @param cbInstr Instruction size.
2214 * @param offNextInstr The offset of the next instruction.
2215 * @param enmEffOpSize Effective operand size.
2216 */
2217DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2218 IEMMODE enmEffOpSize) RT_NOEXCEPT
2219{
2220 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2221 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
2222
2223 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2224 if (enmEffOpSize == IEMMODE_16BIT)
2225 uNewEip &= UINT16_MAX;
2226 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2227 pVCpu->cpum.GstCtx.rip = uNewEip;
2228 else
2229 return iemRaiseGeneralProtectionFault0(pVCpu);
2230
2231#ifndef IEM_WITH_CODE_TLB
2232 iemOpcodeFlushLight(pVCpu, cbInstr);
2233#endif
2234
2235 /*
2236 * Clear RF and finish the instruction (maybe raise #DB).
2237 */
2238 return iemRegFinishClearingRF(pVCpu);
2239}
2240
2241
2242/**
2243 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2244 *
2245 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2246 * segment limit.
2247 *
2248 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2249 * @param cbInstr Instruction size.
2250 * @param offNextInstr The offset of the next instruction.
2251 */
2252DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2253 int8_t offNextInstr) RT_NOEXCEPT
2254{
2255 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2256
2257 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2258 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2259 pVCpu->cpum.GstCtx.rip = uNewIp;
2260 else
2261 return iemRaiseGeneralProtectionFault0(pVCpu);
2262
2263#ifndef IEM_WITH_CODE_TLB
2264 iemOpcodeFlushLight(pVCpu, cbInstr);
2265#endif
2266
2267 /*
2268 * Clear RF and finish the instruction (maybe raise #DB).
2269 */
2270 return iemRegFinishClearingRF(pVCpu);
2271}
2272
2273
2274/**
2275 * Adds a 16-bit signed jump offset to RIP from 64-bit code.
2276 *
2277 * @returns Strict VBox status code.
2278 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2279 * @param cbInstr Instruction size.
2280 * @param offNextInstr The offset of the next instruction.
2281 */
2282DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2283 int16_t offNextInstr) RT_NOEXCEPT
2284{
2285 Assert(IEM_IS_64BIT_CODE(pVCpu));
2286
2287 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2288
2289#ifndef IEM_WITH_CODE_TLB
2290 iemOpcodeFlushLight(pVCpu, cbInstr);
2291#endif
2292
2293 /*
2294 * Clear RF and finish the instruction (maybe raise #DB).
2295 */
2296 return iemRegFinishClearingRF(pVCpu);
2297}
2298
2299
2300/**
2301 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2302 *
2303 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2304 * segment limit.
2305 *
2306 * @returns Strict VBox status code.
2307 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2308 * @param cbInstr Instruction size.
2309 * @param offNextInstr The offset of the next instruction.
2310 *
2311 * @note This is also used by 16-bit code in pre-386 mode, as the code is
2312 * identical.
2313 */
2314DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2315 int16_t offNextInstr) RT_NOEXCEPT
2316{
2317 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2318
2319 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2320 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2321 pVCpu->cpum.GstCtx.rip = uNewIp;
2322 else
2323 return iemRaiseGeneralProtectionFault0(pVCpu);
2324
2325#ifndef IEM_WITH_CODE_TLB
2326 iemOpcodeFlushLight(pVCpu, cbInstr);
2327#endif
2328
2329 /*
2330 * Clear RF and finish the instruction (maybe raise #DB).
2331 */
2332 return iemRegFinishClearingRF(pVCpu);
2333}
2334
2335
2336/**
2337 * Adds a 32-bit signed jump offset to RIP from 64-bit code.
2338 *
2339 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2340 * segment limit.
2341 *
2342 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2343 * only alternative for relative jumps in 64-bit code and that is already
2344 * handled in the decoder stage.
2345 *
2346 * @returns Strict VBox status code.
2347 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2348 * @param cbInstr Instruction size.
2349 * @param offNextInstr The offset of the next instruction.
2350 */
2351DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2352 int32_t offNextInstr) RT_NOEXCEPT
2353{
2354 Assert(IEM_IS_64BIT_CODE(pVCpu));
2355
2356 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2357 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2358 pVCpu->cpum.GstCtx.rip = uNewRip;
2359 else
2360 return iemRaiseGeneralProtectionFault0(pVCpu);
2361
2362#ifndef IEM_WITH_CODE_TLB
2363 iemOpcodeFlushLight(pVCpu, cbInstr);
2364#endif
2365
2366 /*
2367 * Clear RF and finish the instruction (maybe raise #DB).
2368 */
2369 return iemRegFinishClearingRF(pVCpu);
2370}
2371
2372
2373/**
2374 * Adds a 32-bit signed jump offset to RIP from 64-bit code.
2375 *
2376 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2377 * segment limit.
2378 *
2379 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2380 * only alternative for relative jumps in 32-bit code and that is already
2381 * handled in the decoder stage.
2382 *
2383 * @returns Strict VBox status code.
2384 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2385 * @param cbInstr Instruction size.
2386 * @param offNextInstr The offset of the next instruction.
2387 */
2388DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2389 int32_t offNextInstr) RT_NOEXCEPT
2390{
2391 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2392 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2393
2394 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2395 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2396 pVCpu->cpum.GstCtx.rip = uNewEip;
2397 else
2398 return iemRaiseGeneralProtectionFault0(pVCpu);
2399
2400#ifndef IEM_WITH_CODE_TLB
2401 iemOpcodeFlushLight(pVCpu, cbInstr);
2402#endif
2403
2404 /*
2405 * Clear RF and finish the instruction (maybe raise #DB).
2406 */
2407 return iemRegFinishClearingRF(pVCpu);
2408}
2409
2410
2411/**
2412 * Extended version of iemFinishInstructionWithFlagsSet that goes with
2413 * iemRegAddToRipAndFinishingClearingRfEx.
2414 *
2415 * See iemFinishInstructionWithFlagsSet() for details.
2416 */
2417static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2418{
2419 /*
2420 * Raise a #DB.
2421 */
2422 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2423 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2424 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS
2425 | (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2426 /** @todo Do we set all pending \#DB events, or just one? */
2427 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2428 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2429 pVCpu->cpum.GstCtx.rflags.uBoth));
2430 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK);
2431 return iemRaiseDebugException(pVCpu);
2432}
2433
2434
2435/**
2436 * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2437 * others potentially updating EFLAGS.TF.
2438 *
2439 * The single step event must be generated using the TF value at the start of
2440 * the instruction, not the new value set by it.
2441 *
2442 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2443 * @param cbInstr The number of bytes to add.
2444 * @param fEflOld The EFLAGS at the start of the instruction
2445 * execution.
2446 */
2447DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2448{
2449 iemRegAddToRip(pVCpu, cbInstr);
2450 if (!(fEflOld & X86_EFL_TF))
2451 return iemRegFinishClearingRF(pVCpu);
2452 return iemFinishInstructionWithTfSet(pVCpu);
2453}
2454
2455
2456#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2457/**
2458 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2459 *
2460 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2461 */
2462DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2463{
2464 return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2465}
2466#endif
2467
2468
2469/**
2470 * Adds to the stack pointer.
2471 *
2472 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2473 * @param cbToAdd The number of bytes to add (8-bit!).
2474 */
2475DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2476{
2477 if (IEM_IS_64BIT_CODE(pVCpu))
2478 pVCpu->cpum.GstCtx.rsp += cbToAdd;
2479 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2480 pVCpu->cpum.GstCtx.esp += cbToAdd;
2481 else
2482 pVCpu->cpum.GstCtx.sp += cbToAdd;
2483}
2484
2485
2486/**
2487 * Subtracts from the stack pointer.
2488 *
2489 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2490 * @param cbToSub The number of bytes to subtract (8-bit!).
2491 */
2492DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2493{
2494 if (IEM_IS_64BIT_CODE(pVCpu))
2495 pVCpu->cpum.GstCtx.rsp -= cbToSub;
2496 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2497 pVCpu->cpum.GstCtx.esp -= cbToSub;
2498 else
2499 pVCpu->cpum.GstCtx.sp -= cbToSub;
2500}
2501
2502
2503/**
2504 * Adds to the temporary stack pointer.
2505 *
2506 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2507 * @param pTmpRsp The temporary SP/ESP/RSP to update.
2508 * @param cbToAdd The number of bytes to add (16-bit).
2509 */
2510DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2511{
2512 if (IEM_IS_64BIT_CODE(pVCpu))
2513 pTmpRsp->u += cbToAdd;
2514 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2515 pTmpRsp->DWords.dw0 += cbToAdd;
2516 else
2517 pTmpRsp->Words.w0 += cbToAdd;
2518}
2519
2520
2521/**
2522 * Subtracts from the temporary stack pointer.
2523 *
2524 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2525 * @param pTmpRsp The temporary SP/ESP/RSP to update.
2526 * @param cbToSub The number of bytes to subtract.
2527 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
2528 * expecting that.
2529 */
2530DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2531{
2532 if (IEM_IS_64BIT_CODE(pVCpu))
2533 pTmpRsp->u -= cbToSub;
2534 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2535 pTmpRsp->DWords.dw0 -= cbToSub;
2536 else
2537 pTmpRsp->Words.w0 -= cbToSub;
2538}
2539
2540
2541/**
2542 * Calculates the effective stack address for a push of the specified size as
2543 * well as the new RSP value (upper bits may be masked).
2544 *
2545 * @returns Effective stack addressf for the push.
2546 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2547 * @param cbItem The size of the stack item to pop.
2548 * @param puNewRsp Where to return the new RSP value.
2549 */
2550DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2551{
2552 RTUINT64U uTmpRsp;
2553 RTGCPTR GCPtrTop;
2554 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2555
2556 if (IEM_IS_64BIT_CODE(pVCpu))
2557 GCPtrTop = uTmpRsp.u -= cbItem;
2558 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2559 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2560 else
2561 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2562 *puNewRsp = uTmpRsp.u;
2563 return GCPtrTop;
2564}
2565
2566
2567/**
2568 * Gets the current stack pointer and calculates the value after a pop of the
2569 * specified size.
2570 *
2571 * @returns Current stack pointer.
2572 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2573 * @param cbItem The size of the stack item to pop.
2574 * @param puNewRsp Where to return the new RSP value.
2575 */
2576DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2577{
2578 RTUINT64U uTmpRsp;
2579 RTGCPTR GCPtrTop;
2580 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2581
2582 if (IEM_IS_64BIT_CODE(pVCpu))
2583 {
2584 GCPtrTop = uTmpRsp.u;
2585 uTmpRsp.u += cbItem;
2586 }
2587 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2588 {
2589 GCPtrTop = uTmpRsp.DWords.dw0;
2590 uTmpRsp.DWords.dw0 += cbItem;
2591 }
2592 else
2593 {
2594 GCPtrTop = uTmpRsp.Words.w0;
2595 uTmpRsp.Words.w0 += cbItem;
2596 }
2597 *puNewRsp = uTmpRsp.u;
2598 return GCPtrTop;
2599}
2600
2601
2602/**
2603 * Calculates the effective stack address for a push of the specified size as
2604 * well as the new temporary RSP value (upper bits may be masked).
2605 *
2606 * @returns Effective stack addressf for the push.
2607 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2608 * @param pTmpRsp The temporary stack pointer. This is updated.
2609 * @param cbItem The size of the stack item to pop.
2610 */
2611DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2612{
2613 RTGCPTR GCPtrTop;
2614
2615 if (IEM_IS_64BIT_CODE(pVCpu))
2616 GCPtrTop = pTmpRsp->u -= cbItem;
2617 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2618 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2619 else
2620 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2621 return GCPtrTop;
2622}
2623
2624
2625/**
2626 * Gets the effective stack address for a pop of the specified size and
2627 * calculates and updates the temporary RSP.
2628 *
2629 * @returns Current stack pointer.
2630 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2631 * @param pTmpRsp The temporary stack pointer. This is updated.
2632 * @param cbItem The size of the stack item to pop.
2633 */
2634DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2635{
2636 RTGCPTR GCPtrTop;
2637 if (IEM_IS_64BIT_CODE(pVCpu))
2638 {
2639 GCPtrTop = pTmpRsp->u;
2640 pTmpRsp->u += cbItem;
2641 }
2642 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2643 {
2644 GCPtrTop = pTmpRsp->DWords.dw0;
2645 pTmpRsp->DWords.dw0 += cbItem;
2646 }
2647 else
2648 {
2649 GCPtrTop = pTmpRsp->Words.w0;
2650 pTmpRsp->Words.w0 += cbItem;
2651 }
2652 return GCPtrTop;
2653}
2654
2655/** @} */
2656
2657
2658/** @name FPU access and helpers.
2659 *
2660 * @{
2661 */
2662
2663
2664/**
2665 * Hook for preparing to use the host FPU.
2666 *
2667 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2668 *
2669 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2670 */
2671DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2672{
2673#ifdef IN_RING3
2674 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2675#else
2676 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2677#endif
2678 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2679}
2680
2681
2682/**
2683 * Hook for preparing to use the host FPU for SSE.
2684 *
2685 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2686 *
2687 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2688 */
2689DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2690{
2691 iemFpuPrepareUsage(pVCpu);
2692}
2693
2694
2695/**
2696 * Hook for preparing to use the host FPU for AVX.
2697 *
2698 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2699 *
2700 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2701 */
2702DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2703{
2704 iemFpuPrepareUsage(pVCpu);
2705}
2706
2707
2708/**
2709 * Hook for actualizing the guest FPU state before the interpreter reads it.
2710 *
2711 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2712 *
2713 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2714 */
2715DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2716{
2717#ifdef IN_RING3
2718 NOREF(pVCpu);
2719#else
2720 CPUMRZFpuStateActualizeForRead(pVCpu);
2721#endif
2722 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2723}
2724
2725
2726/**
2727 * Hook for actualizing the guest FPU state before the interpreter changes it.
2728 *
2729 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2730 *
2731 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2732 */
2733DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2734{
2735#ifdef IN_RING3
2736 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2737#else
2738 CPUMRZFpuStateActualizeForChange(pVCpu);
2739#endif
2740 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2741}
2742
2743
2744/**
2745 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2746 * only.
2747 *
2748 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2749 *
2750 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2751 */
2752DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2753{
2754#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
2755 NOREF(pVCpu);
2756#else
2757 CPUMRZFpuStateActualizeSseForRead(pVCpu);
2758#endif
2759 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2760}
2761
2762
2763/**
2764 * Hook for actualizing the guest XMM0..15 and MXCSR register state for
2765 * read+write.
2766 *
2767 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2768 *
2769 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2770 */
2771DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2772{
2773#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
2774 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2775#else
2776 CPUMRZFpuStateActualizeForChange(pVCpu);
2777#endif
2778 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2779
2780 /* Make sure any changes are loaded the next time around. */
2781 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE;
2782}
2783
2784
2785/**
2786 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2787 * only.
2788 *
2789 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2790 *
2791 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2792 */
2793DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2794{
2795#ifdef IN_RING3
2796 NOREF(pVCpu);
2797#else
2798 CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2799#endif
2800 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2801}
2802
2803
2804/**
2805 * Hook for actualizing the guest YMM0..15 and MXCSR register state for
2806 * read+write.
2807 *
2808 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2809 *
2810 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2811 */
2812DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2813{
2814#ifdef IN_RING3
2815 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2816#else
2817 CPUMRZFpuStateActualizeForChange(pVCpu);
2818#endif
2819 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2820
2821 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2822 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;
2823}
2824
2825
2826/**
2827 * Stores a QNaN value into a FPU register.
2828 *
2829 * @param pReg Pointer to the register.
2830 */
2831DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2832{
2833 pReg->au32[0] = UINT32_C(0x00000000);
2834 pReg->au32[1] = UINT32_C(0xc0000000);
2835 pReg->au16[4] = UINT16_C(0xffff);
2836}
2837
2838
2839/**
2840 * Updates the FOP, FPU.CS and FPUIP registers, extended version.
2841 *
2842 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2843 * @param pFpuCtx The FPU context.
2844 * @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2845 */
2846DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2847{
2848 Assert(uFpuOpcode != UINT16_MAX);
2849 pFpuCtx->FOP = uFpuOpcode;
2850 /** @todo x87.CS and FPUIP needs to be kept seperately. */
2851 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2852 {
2853 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2854 * happens in real mode here based on the fnsave and fnstenv images. */
2855 pFpuCtx->CS = 0;
2856 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2857 }
2858 else if (!IEM_IS_LONG_MODE(pVCpu))
2859 {
2860 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2861 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2862 }
2863 else
2864 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2865}
2866
2867
2868/**
2869 * Marks the specified stack register as free (for FFREE).
2870 *
2871 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2872 * @param iStReg The register to free.
2873 */
2874DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2875{
2876 Assert(iStReg < 8);
2877 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2878 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2879 pFpuCtx->FTW &= ~RT_BIT(iReg);
2880}
2881
2882
2883/**
2884 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2885 *
2886 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2887 */
2888DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2889{
2890 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2891 uint16_t uFsw = pFpuCtx->FSW;
2892 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2893 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2894 uFsw &= ~X86_FSW_TOP_MASK;
2895 uFsw |= uTop;
2896 pFpuCtx->FSW = uFsw;
2897}
2898
2899
2900/**
2901 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2902 *
2903 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2904 */
2905DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2906{
2907 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2908 uint16_t uFsw = pFpuCtx->FSW;
2909 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2910 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2911 uFsw &= ~X86_FSW_TOP_MASK;
2912 uFsw |= uTop;
2913 pFpuCtx->FSW = uFsw;
2914}
2915
2916
2917
2918
2919DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2920{
2921 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2922 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2923 if (pFpuCtx->FTW & RT_BIT(iReg))
2924 return VINF_SUCCESS;
2925 return VERR_NOT_FOUND;
2926}
2927
2928
2929DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2930{
2931 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2932 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2933 if (pFpuCtx->FTW & RT_BIT(iReg))
2934 {
2935 *ppRef = &pFpuCtx->aRegs[iStReg].r80;
2936 return VINF_SUCCESS;
2937 }
2938 return VERR_NOT_FOUND;
2939}
2940
2941
2942DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2943 uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2944{
2945 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2946 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2947 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2948 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2949 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
2950 {
2951 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2952 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2953 return VINF_SUCCESS;
2954 }
2955 return VERR_NOT_FOUND;
2956}
2957
2958
2959DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2960{
2961 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2962 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2963 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2964 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2965 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
2966 {
2967 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2968 return VINF_SUCCESS;
2969 }
2970 return VERR_NOT_FOUND;
2971}
2972
2973
2974/**
2975 * Rotates the stack registers when setting new TOS.
2976 *
2977 * @param pFpuCtx The FPU context.
2978 * @param iNewTop New TOS value.
2979 * @remarks We only do this to speed up fxsave/fxrstor which
2980 * arrange the FP registers in stack order.
2981 * MUST be done before writing the new TOS (FSW).
2982 */
2983DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2984{
2985 uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2986 RTFLOAT80U ar80Temp[8];
2987
2988 if (iOldTop == iNewTop)
2989 return;
2990
2991 /* Unscrew the stack and get it into 'native' order. */
2992 ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2993 ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2994 ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2995 ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2996 ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2997 ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2998 ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2999 ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3000
3001 /* Now rotate the stack to the new position. */
3002 pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3003 pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3004 pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3005 pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3006 pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3007 pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3008 pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3009 pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3010}
3011
3012
3013/**
3014 * Updates the FPU exception status after FCW is changed.
3015 *
3016 * @param pFpuCtx The FPU context.
3017 */
3018DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3019{
3020 uint16_t u16Fsw = pFpuCtx->FSW;
3021 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3022 u16Fsw |= X86_FSW_ES | X86_FSW_B;
3023 else
3024 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
3025 pFpuCtx->FSW = u16Fsw;
3026}
3027
3028
3029/**
3030 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3031 *
3032 * @returns The full FTW.
3033 * @param pFpuCtx The FPU context.
3034 */
3035DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3036{
3037 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3038 uint16_t u16Ftw = 0;
3039 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3040 for (unsigned iSt = 0; iSt < 8; iSt++)
3041 {
3042 unsigned const iReg = (iSt + iTop) & 7;
3043 if (!(u8Ftw & RT_BIT(iReg)))
3044 u16Ftw |= 3 << (iReg * 2); /* empty */
3045 else
3046 {
3047 uint16_t uTag;
3048 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3049 if (pr80Reg->s.uExponent == 0x7fff)
3050 uTag = 2; /* Exponent is all 1's => Special. */
3051 else if (pr80Reg->s.uExponent == 0x0000)
3052 {
3053 if (pr80Reg->s.uMantissa == 0x0000)
3054 uTag = 1; /* All bits are zero => Zero. */
3055 else
3056 uTag = 2; /* Must be special. */
3057 }
3058 else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3059 uTag = 0; /* Valid. */
3060 else
3061 uTag = 2; /* Must be special. */
3062
3063 u16Ftw |= uTag << (iReg * 2);
3064 }
3065 }
3066
3067 return u16Ftw;
3068}
3069
3070
3071/**
3072 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3073 *
3074 * @returns The compressed FTW.
3075 * @param u16FullFtw The full FTW to convert.
3076 */
3077DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3078{
3079 uint8_t u8Ftw = 0;
3080 for (unsigned i = 0; i < 8; i++)
3081 {
3082 if ((u16FullFtw & 3) != 3 /*empty*/)
3083 u8Ftw |= RT_BIT(i);
3084 u16FullFtw >>= 2;
3085 }
3086
3087 return u8Ftw;
3088}
3089
3090/** @} */
3091
3092
3093/** @name Memory access.
3094 *
3095 * @{
3096 */
3097
3098
3099/**
3100 * Checks whether alignment checks are enabled or not.
3101 *
3102 * @returns true if enabled, false if not.
3103 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3104 */
3105DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3106{
3107 AssertCompile(X86_CR0_AM == X86_EFL_AC);
3108 return IEM_GET_CPL(pVCpu) == 3
3109 && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3110}
3111
3112/**
3113 * Checks if the given segment can be written to, raise the appropriate
3114 * exception if not.
3115 *
3116 * @returns VBox strict status code.
3117 *
3118 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3119 * @param pHid Pointer to the hidden register.
3120 * @param iSegReg The register number.
3121 * @param pu64BaseAddr Where to return the base address to use for the
3122 * segment. (In 64-bit code it may differ from the
3123 * base in the hidden segment.)
3124 */
3125DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3126 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3127{
3128 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3129
3130 if (IEM_IS_64BIT_CODE(pVCpu))
3131 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3132 else
3133 {
3134 if (!pHid->Attr.n.u1Present)
3135 {
3136 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3137 AssertRelease(uSel == 0);
3138 Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3139 return iemRaiseGeneralProtectionFault0(pVCpu);
3140 }
3141
3142 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3143 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3144 && !IEM_IS_64BIT_CODE(pVCpu) )
3145 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3146 *pu64BaseAddr = pHid->u64Base;
3147 }
3148 return VINF_SUCCESS;
3149}
3150
3151
3152/**
3153 * Checks if the given segment can be read from, raise the appropriate
3154 * exception if not.
3155 *
3156 * @returns VBox strict status code.
3157 *
3158 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3159 * @param pHid Pointer to the hidden register.
3160 * @param iSegReg The register number.
3161 * @param pu64BaseAddr Where to return the base address to use for the
3162 * segment. (In 64-bit code it may differ from the
3163 * base in the hidden segment.)
3164 */
3165DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3166 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3167{
3168 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3169
3170 if (IEM_IS_64BIT_CODE(pVCpu))
3171 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3172 else
3173 {
3174 if (!pHid->Attr.n.u1Present)
3175 {
3176 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3177 AssertRelease(uSel == 0);
3178 Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3179 return iemRaiseGeneralProtectionFault0(pVCpu);
3180 }
3181
3182 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3183 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3184 *pu64BaseAddr = pHid->u64Base;
3185 }
3186 return VINF_SUCCESS;
3187}
3188
3189
3190/**
3191 * Maps a physical page.
3192 *
3193 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3194 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3195 * @param GCPhysMem The physical address.
3196 * @param fAccess The intended access.
3197 * @param ppvMem Where to return the mapping address.
3198 * @param pLock The PGM lock.
3199 */
3200DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3201 void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3202{
3203#ifdef IEM_LOG_MEMORY_WRITES
3204 if (fAccess & IEM_ACCESS_TYPE_WRITE)
3205 return VERR_PGM_PHYS_TLB_CATCH_ALL;
3206#endif
3207
3208 /** @todo This API may require some improving later. A private deal with PGM
3209 * regarding locking and unlocking needs to be struct. A couple of TLBs
3210 * living in PGM, but with publicly accessible inlined access methods
3211 * could perhaps be an even better solution. */
3212 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3213 GCPhysMem,
3214 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3215 RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3216 ppvMem,
3217 pLock);
3218 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
3219 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3220
3221 return rc;
3222}
3223
3224
3225/**
3226 * Unmap a page previously mapped by iemMemPageMap.
3227 *
3228 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3229 * @param GCPhysMem The physical address.
3230 * @param fAccess The intended access.
3231 * @param pvMem What iemMemPageMap returned.
3232 * @param pLock The PGM lock.
3233 */
3234DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3235 const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3236{
3237 NOREF(pVCpu);
3238 NOREF(GCPhysMem);
3239 NOREF(fAccess);
3240 NOREF(pvMem);
3241 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3242}
3243
3244#ifdef IEM_WITH_SETJMP
3245
3246/** @todo slim this down */
3247DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3248 size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3249{
3250 Assert(cbMem >= 1);
3251 Assert(iSegReg < X86_SREG_COUNT);
3252
3253 /*
3254 * 64-bit mode is simpler.
3255 */
3256 if (IEM_IS_64BIT_CODE(pVCpu))
3257 {
3258 if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3259 {
3260 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3261 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3262 GCPtrMem += pSel->u64Base;
3263 }
3264
3265 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3266 return GCPtrMem;
3267 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3268 }
3269 /*
3270 * 16-bit and 32-bit segmentation.
3271 */
3272 else if (iSegReg != UINT8_MAX)
3273 {
3274 /** @todo Does this apply to segments with 4G-1 limit? */
3275 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3276 if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3277 {
3278 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3279 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3280 switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
3281 | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */
3282 | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */
3283 | X86_SEL_TYPE_CODE))
3284 {
3285 case X86DESCATTR_P: /* readonly data, expand up */
3286 case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */
3287 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */
3288 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */
3289 /* expand up */
3290 if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3291 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3292 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3293 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3294 break;
3295
3296 case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3297 case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */
3298 /* expand down */
3299 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3300 && ( pSel->Attr.n.u1DefBig
3301 || GCPtrLast32 <= UINT32_C(0xffff)) ))
3302 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3303 Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3304 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3305 break;
3306
3307 default:
3308 Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3309 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3310 break;
3311 }
3312 }
3313 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3314 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3315 }
3316 /*
3317 * 32-bit flat address.
3318 */
3319 else
3320 return GCPtrMem;
3321}
3322
3323
3324/** @todo slim this down */
3325DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3326 RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3327{
3328 Assert(cbMem >= 1);
3329 Assert(iSegReg < X86_SREG_COUNT);
3330
3331 /*
3332 * 64-bit mode is simpler.
3333 */
3334 if (IEM_IS_64BIT_CODE(pVCpu))
3335 {
3336 if (iSegReg >= X86_SREG_FS)
3337 {
3338 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3339 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3340 GCPtrMem += pSel->u64Base;
3341 }
3342
3343 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3344 return GCPtrMem;
3345 }
3346 /*
3347 * 16-bit and 32-bit segmentation.
3348 */
3349 else
3350 {
3351 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3352 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3353 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
3354 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
3355 if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE)) /* data, expand up */
3356 {
3357 /* expand up */
3358 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
3359 if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
3360 && GCPtrLast32 > (uint32_t)GCPtrMem))
3361 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3362 }
3363 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
3364 {
3365 /* expand down */
3366 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
3367 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3368 && GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
3369 && GCPtrLast32 > (uint32_t)GCPtrMem))
3370 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3371 }
3372 else
3373 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3374 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3375 }
3376 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3377}
3378
3379#endif /* IEM_WITH_SETJMP */
3380
3381/**
3382 * Fakes a long mode stack selector for SS = 0.
3383 *
3384 * @param pDescSs Where to return the fake stack descriptor.
3385 * @param uDpl The DPL we want.
3386 */
3387DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3388{
3389 pDescSs->Long.au64[0] = 0;
3390 pDescSs->Long.au64[1] = 0;
3391 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3392 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3393 pDescSs->Long.Gen.u2Dpl = uDpl;
3394 pDescSs->Long.Gen.u1Present = 1;
3395 pDescSs->Long.Gen.u1Long = 1;
3396}
3397
3398/** @} */
3399
3400
3401#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3402
3403/**
3404 * Gets CR0 fixed-0 bits in VMX operation.
3405 *
3406 * We do this rather than fetching what we report to the guest (in
3407 * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3408 * values regardless of whether unrestricted-guest feature is available on the CPU.
3409 *
3410 * @returns CR0 fixed-0 bits.
3411 * @param pVCpu The cross context virtual CPU structure.
3412 * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3413 * must be returned. When @c false, the CR0 fixed-0
3414 * bits for VMX root mode is returned.
3415 *
3416 */
3417DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3418{
3419 Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3420
3421 PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3422 if ( fVmxNonRootMode
3423 && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3424 return VMX_V_CR0_FIXED0_UX;
3425 return VMX_V_CR0_FIXED0;
3426}
3427
3428
3429/**
3430 * Sets virtual-APIC write emulation as pending.
3431 *
3432 * @param pVCpu The cross context virtual CPU structure.
3433 * @param offApic The offset in the virtual-APIC page that was written.
3434 */
3435DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3436{
3437 Assert(offApic < XAPIC_OFF_END + 4);
3438
3439 /*
3440 * Record the currently updated APIC offset, as we need this later for figuring
3441 * out whether to perform TPR, EOI or self-IPI virtualization as well as well
3442 * as for supplying the exit qualification when causing an APIC-write VM-exit.
3443 */
3444 pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3445
3446 /*
3447 * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3448 * virtualization or APIC-write emulation).
3449 */
3450 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3451 VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3452}
3453
3454#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3455
3456#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette