VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 100859

Last change on this file since 100859 was 100850, checked in by vboxsync, 16 months ago

VMM/IEM: Combined the with-in-page and alignment checks for the inlined code, making for a straigher code path for correctly aligned data accesses. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 123.8 KB
Line 
1/* $Id: IEMInline.h 100850 2023-08-10 12:04:43Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Inlined Functions.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29#define VMM_INCLUDED_SRC_include_IEMInline_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35
36/**
37 * Makes status code addjustments (pass up from I/O and access handler)
38 * as well as maintaining statistics.
39 *
40 * @returns Strict VBox status code to pass up.
41 * @param pVCpu The cross context virtual CPU structure of the calling thread.
42 * @param rcStrict The status from executing an instruction.
43 */
44DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45{
46 if (rcStrict != VINF_SUCCESS)
47 {
48 /* Deal with the cases that should be treated as VINF_SUCCESS first. */
49 if ( rcStrict == VINF_IEM_YIELD_PENDING_FF
50#ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */
51 || rcStrict == VINF_VMX_VMEXIT
52#endif
53#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
54 || rcStrict == VINF_SVM_VMEXIT
55#endif
56 )
57 {
58 if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
59 rcStrict = VINF_SUCCESS;
60 else
61 {
62 pVCpu->iem.s.cRetPassUpStatus++;
63 rcStrict = pVCpu->iem.s.rcPassUp;
64 }
65 }
66 else if (RT_SUCCESS(rcStrict))
67 {
68 AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
69 || rcStrict == VINF_IOM_R3_IOPORT_READ
70 || rcStrict == VINF_IOM_R3_IOPORT_WRITE
71 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
72 || rcStrict == VINF_IOM_R3_MMIO_READ
73 || rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
74 || rcStrict == VINF_IOM_R3_MMIO_WRITE
75 || rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
76 || rcStrict == VINF_CPUM_R3_MSR_READ
77 || rcStrict == VINF_CPUM_R3_MSR_WRITE
78 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
79 || rcStrict == VINF_EM_RAW_TO_R3
80 || rcStrict == VINF_EM_TRIPLE_FAULT
81 || rcStrict == VINF_GIM_R3_HYPERCALL
82 /* raw-mode / virt handlers only: */
83 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
84 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
85 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
86 || rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
87 || rcStrict == VINF_SELM_SYNC_GDT
88 || rcStrict == VINF_CSAM_PENDING_ACTION
89 || rcStrict == VINF_PATM_CHECK_PATCH_PAGE
90 /* nested hw.virt codes: */
91 || rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
92 || rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
93 , ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
94/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
95 int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
96 if (rcPassUp == VINF_SUCCESS)
97 pVCpu->iem.s.cRetInfStatuses++;
98 else if ( rcPassUp < VINF_EM_FIRST
99 || rcPassUp > VINF_EM_LAST
100 || rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
101 {
102 Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
103 pVCpu->iem.s.cRetPassUpStatus++;
104 rcStrict = rcPassUp;
105 }
106 else
107 {
108 Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
109 pVCpu->iem.s.cRetInfStatuses++;
110 }
111 }
112 else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
113 pVCpu->iem.s.cRetAspectNotImplemented++;
114 else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
115 pVCpu->iem.s.cRetInstrNotImplemented++;
116 else
117 pVCpu->iem.s.cRetErrStatuses++;
118 }
119 else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
120 {
121 pVCpu->iem.s.cRetPassUpStatus++;
122 rcStrict = pVCpu->iem.s.rcPassUp;
123 }
124
125 /* Just clear it here as well. */
126 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
127
128 return rcStrict;
129}
130
131
132/**
133 * Sets the pass up status.
134 *
135 * @returns VINF_SUCCESS.
136 * @param pVCpu The cross context virtual CPU structure of the
137 * calling thread.
138 * @param rcPassUp The pass up status. Must be informational.
139 * VINF_SUCCESS is not allowed.
140 */
141DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
142{
143 AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
144
145 int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
146 if (rcOldPassUp == VINF_SUCCESS)
147 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148 /* If both are EM scheduling codes, use EM priority rules. */
149 else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
150 && rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
151 {
152 if (rcPassUp < rcOldPassUp)
153 {
154 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
155 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
156 }
157 else
158 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
159 }
160 /* Override EM scheduling with specific status code. */
161 else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
162 {
163 Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
164 pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
165 }
166 /* Don't override specific status code, first come first served. */
167 else
168 Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
169 return VINF_SUCCESS;
170}
171
172
173/**
174 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
175 *
176 * Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
177 * reject expand down data segments and conforming code segments.
178 *
179 * ASSUMES that the CPU is in 32-bit mode.
180 *
181 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
182 * @param pVCpu The cross context virtual CPU structure of the
183 * calling thread.
184 * @sa iemCalc32BitFlatIndicatorEsDs
185 */
186DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
187{
188 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
189 return ( ( pVCpu->cpum.GstCtx.es.Attr.u
190 | pVCpu->cpum.GstCtx.cs.Attr.u
191 | pVCpu->cpum.GstCtx.ss.Attr.u
192 | pVCpu->cpum.GstCtx.ds.Attr.u)
193 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE))
194 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)
195 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
196 | (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
197 | (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
198 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
199 == 0
200 && ( pVCpu->cpum.GstCtx.es.u64Base
201 | pVCpu->cpum.GstCtx.cs.u64Base
202 | pVCpu->cpum.GstCtx.ss.u64Base
203 | pVCpu->cpum.GstCtx.ds.u64Base)
204 == 0
205 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))
206 ? IEM_F_MODE_X86_32BIT_FLAT : 0;
207}
208
209
210/**
211 * Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
212 * flat already.
213 *
214 * This is used by sysenter.
215 *
216 * @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
217 * @param pVCpu The cross context virtual CPU structure of the
218 * calling thread.
219 * @sa iemCalc32BitFlatIndicator
220 */
221DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
222{
223 AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
224 return ( ( pVCpu->cpum.GstCtx.es.Attr.u
225 | pVCpu->cpum.GstCtx.ds.Attr.u)
226 & (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86DESCATTR_UNUSABLE))
227 == (X86_SEL_TYPE_ACCESSED | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_P)
228 && ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
229 | (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
230 == 0
231 && ( pVCpu->cpum.GstCtx.es.u64Base
232 | pVCpu->cpum.GstCtx.ds.u64Base)
233 == 0
234 && !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_ES))
235 ? IEM_F_MODE_X86_32BIT_FLAT : 0;
236}
237
238
239/**
240 * Calculates the IEM_F_MODE_XXX and CPL flags.
241 *
242 * @returns IEM_F_MODE_XXX
243 * @param pVCpu The cross context virtual CPU structure of the
244 * calling thread.
245 */
246DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
247{
248 /*
249 * We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
250 * here to try get this done as efficiently as possible.
251 */
252 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS);
253
254 if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
255 {
256 if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
257 {
258 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
259 uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
260 if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
261 {
262 Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
263 fExec |= IEM_F_MODE_X86_32BIT_PROT | iemCalc32BitFlatIndicator(pVCpu);
264 }
265 else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
266 && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
267 fExec |= IEM_F_MODE_X86_64BIT;
268 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
269 fExec |= IEM_F_MODE_X86_16BIT_PROT;
270 else
271 fExec |= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
272 return fExec;
273 }
274 return IEM_F_MODE_X86_16BIT_PROT_V86 | (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
275 }
276
277 /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
278 if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
279 {
280 if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
281 return IEM_F_MODE_X86_16BIT;
282 return IEM_F_MODE_X86_16BIT_PRE_386;
283 }
284
285 /* 32-bit unreal mode. */
286 return IEM_F_MODE_X86_32BIT | iemCalc32BitFlatIndicator(pVCpu);
287}
288
289
290/**
291 * Calculates the AMD-V and VT-x related context flags.
292 *
293 * @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
294 * IEM_F_X86_CTX_VMX.
295 * @param pVCpu The cross context virtual CPU structure of the
296 * calling thread.
297 */
298DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
299{
300 /*
301 * This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
302 * and CPUMIsGuestInNestedHwvirtMode to some extent.
303 */
304 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
305
306 AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
307 uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
308 | (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
309 if (RT_LIKELY(!fTmp))
310 return 0; /* likely */
311
312 if (fTmp & X86_CR4_VMXE)
313 {
314 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
315 if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
316 return IEM_F_X86_CTX_VMX | IEM_F_X86_CTX_IN_GUEST;
317 return IEM_F_X86_CTX_VMX;
318 }
319
320 Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
321 if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
322 return IEM_F_X86_CTX_SVM | IEM_F_X86_CTX_IN_GUEST;
323 return IEM_F_X86_CTX_SVM;
324}
325
326
327/**
328 * Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
329 *
330 * @returns IEM_F_BRK_PENDING_XXX or zero.
331 * @param pVCpu The cross context virtual CPU structure of the
332 * calling thread.
333 */
334DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
335{
336 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
337
338 if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
339 && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
340 return 0;
341 return iemCalcExecDbgFlagsSlow(pVCpu);
342}
343
344/**
345 * Calculates the the IEM_F_XXX flags.
346 *
347 * @returns IEM_F_XXX combination match the current CPU state.
348 * @param pVCpu The cross context virtual CPU structure of the
349 * calling thread.
350 */
351DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
352{
353 return iemCalcExecModeAndCplFlags(pVCpu)
354 | iemCalcExecHwVirtFlags(pVCpu)
355 /* SMM is not yet implemented */
356 | iemCalcExecDbgFlags(pVCpu)
357 ;
358}
359
360
361/**
362 * Re-calculates the MODE and CPL parts of IEMCPU::fExec.
363 *
364 * @param pVCpu The cross context virtual CPU structure of the
365 * calling thread.
366 */
367DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
368{
369 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK | IEM_F_X86_CPL_MASK))
370 | iemCalcExecModeAndCplFlags(pVCpu);
371}
372
373
374/**
375 * Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
376 *
377 * @param pVCpu The cross context virtual CPU structure of the
378 * calling thread.
379 */
380DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
381{
382 pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
383 | iemCalcExecDbgFlags(pVCpu);
384}
385
386
387#ifndef IEM_WITH_OPAQUE_DECODER_STATE
388
389# if defined(VBOX_INCLUDED_vmm_dbgf_h) || defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
390/**
391 * Initializes the execution state.
392 *
393 * @param pVCpu The cross context virtual CPU structure of the
394 * calling thread.
395 * @param fExecOpts Optional execution flags:
396 * - IEM_F_BYPASS_HANDLERS
397 * - IEM_F_X86_DISREGARD_LOCK
398 *
399 * @remarks Callers of this must call iemUninitExec() to undo potentially fatal
400 * side-effects in strict builds.
401 */
402DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
403{
404 IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
405 Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
406 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
407 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
408 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
409 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
410 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
411 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
412 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
413 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
414
415 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
416 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | fExecOpts;
417 pVCpu->iem.s.cActiveMappings = 0;
418 pVCpu->iem.s.iNextMapping = 0;
419
420# ifdef VBOX_STRICT
421 pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
422 pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
423 pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
424 pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
425 pVCpu->iem.s.fPrefixes = 0xfeedbeef;
426 pVCpu->iem.s.uRexReg = 127;
427 pVCpu->iem.s.uRexB = 127;
428 pVCpu->iem.s.offModRm = 127;
429 pVCpu->iem.s.uRexIndex = 127;
430 pVCpu->iem.s.iEffSeg = 127;
431 pVCpu->iem.s.idxPrefix = 127;
432 pVCpu->iem.s.uVex3rdReg = 127;
433 pVCpu->iem.s.uVexLength = 127;
434 pVCpu->iem.s.fEvexStuff = 127;
435 pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
436# ifdef IEM_WITH_CODE_TLB
437 pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
438 pVCpu->iem.s.pbInstrBuf = NULL;
439 pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
440 pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
441 pVCpu->iem.s.offCurInstrStart = INT16_MAX;
442 pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
443# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
444 pVCpu->iem.s.offOpcode = 127;
445# endif
446# else
447 pVCpu->iem.s.offOpcode = 127;
448 pVCpu->iem.s.cbOpcode = 127;
449# endif
450# endif /* VBOX_STRICT */
451}
452# endif /* VBOX_INCLUDED_vmm_dbgf_h */
453
454
455# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
456/**
457 * Performs a minimal reinitialization of the execution state.
458 *
459 * This is intended to be used by VM-exits, SMM, LOADALL and other similar
460 * 'world-switch' types operations on the CPU. Currently only nested
461 * hardware-virtualization uses it.
462 *
463 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
464 * @param cbInstr The instruction length (for flushing).
465 */
466DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
467{
468 pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) | (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
469 iemOpcodeFlushHeavy(pVCpu, cbInstr);
470}
471# endif
472
473
474/**
475 * Counterpart to #iemInitExec that undoes evil strict-build stuff.
476 *
477 * @param pVCpu The cross context virtual CPU structure of the
478 * calling thread.
479 */
480DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
481{
482 /* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
483# ifdef VBOX_STRICT
484# ifdef IEM_WITH_CODE_TLB
485 NOREF(pVCpu);
486# else
487 pVCpu->iem.s.cbOpcode = 0;
488# endif
489# else
490 NOREF(pVCpu);
491# endif
492}
493
494
495/**
496 * Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
497 *
498 * Only calling iemRCRawMaybeReenter in raw-mode, obviously.
499 *
500 * @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
501 * @param pVCpu The cross context virtual CPU structure of the calling thread.
502 * @param rcStrict The status code to fiddle.
503 */
504DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
505{
506 iemUninitExec(pVCpu);
507 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
508}
509
510
511/**
512 * Macro used by the IEMExec* method to check the given instruction length.
513 *
514 * Will return on failure!
515 *
516 * @param a_cbInstr The given instruction length.
517 * @param a_cbMin The minimum length.
518 */
519# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
520 AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
521 ("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
522
523
524# ifndef IEM_WITH_SETJMP
525
526/**
527 * Fetches the first opcode byte.
528 *
529 * @returns Strict VBox status code.
530 * @param pVCpu The cross context virtual CPU structure of the
531 * calling thread.
532 * @param pu8 Where to return the opcode byte.
533 */
534DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
535{
536 /*
537 * Check for hardware instruction breakpoints.
538 */
539 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
540 { /* likely */ }
541 else
542 {
543 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
544 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
545 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
546 { /* likely */ }
547 else
548 {
549 *pu8 = 0xff; /* shut up gcc. sigh */
550 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
551 return iemRaiseDebugException(pVCpu);
552 return rcStrict;
553 }
554 }
555
556 /*
557 * Fetch the first opcode byte.
558 */
559 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
560 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
561 {
562 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
563 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
564 return VINF_SUCCESS;
565 }
566 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
567}
568
569# else /* IEM_WITH_SETJMP */
570
571/**
572 * Fetches the first opcode byte, longjmp on error.
573 *
574 * @returns The opcode byte.
575 * @param pVCpu The cross context virtual CPU structure of the calling thread.
576 */
577DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
578{
579 /*
580 * Check for hardware instruction breakpoints.
581 */
582 if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
583 { /* likely */ }
584 else
585 {
586 VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
587 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
588 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
589 { /* likely */ }
590 else
591 {
592 if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
593 rcStrict = iemRaiseDebugException(pVCpu);
594 IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
595 }
596 }
597
598 /*
599 * Fetch the first opcode byte.
600 */
601# ifdef IEM_WITH_CODE_TLB
602 uint8_t bRet;
603 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
604 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
605 if (RT_LIKELY( pbBuf != NULL
606 && offBuf < pVCpu->iem.s.cbInstrBuf))
607 {
608 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
609 bRet = pbBuf[offBuf];
610 }
611 else
612 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
613# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
614 Assert(pVCpu->iem.s.offOpcode == 0);
615 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
616# endif
617 return bRet;
618
619# else /* !IEM_WITH_CODE_TLB */
620 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
621 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
622 {
623 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
624 return pVCpu->iem.s.abOpcode[offOpcode];
625 }
626 return iemOpcodeGetNextU8SlowJmp(pVCpu);
627# endif
628}
629
630# endif /* IEM_WITH_SETJMP */
631
632/**
633 * Fetches the first opcode byte, returns/throws automatically on failure.
634 *
635 * @param a_pu8 Where to return the opcode byte.
636 * @remark Implicitly references pVCpu.
637 */
638# ifndef IEM_WITH_SETJMP
639# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
640 do \
641 { \
642 VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
643 if (rcStrict2 == VINF_SUCCESS) \
644 { /* likely */ } \
645 else \
646 return rcStrict2; \
647 } while (0)
648# else
649# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
650# endif /* IEM_WITH_SETJMP */
651
652
653# ifndef IEM_WITH_SETJMP
654
655/**
656 * Fetches the next opcode byte.
657 *
658 * @returns Strict VBox status code.
659 * @param pVCpu The cross context virtual CPU structure of the
660 * calling thread.
661 * @param pu8 Where to return the opcode byte.
662 */
663DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
664{
665 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
666 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
667 {
668 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
669 *pu8 = pVCpu->iem.s.abOpcode[offOpcode];
670 return VINF_SUCCESS;
671 }
672 return iemOpcodeGetNextU8Slow(pVCpu, pu8);
673}
674
675# else /* IEM_WITH_SETJMP */
676
677/**
678 * Fetches the next opcode byte, longjmp on error.
679 *
680 * @returns The opcode byte.
681 * @param pVCpu The cross context virtual CPU structure of the calling thread.
682 */
683DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
684{
685# ifdef IEM_WITH_CODE_TLB
686 uint8_t bRet;
687 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
688 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
689 if (RT_LIKELY( pbBuf != NULL
690 && offBuf < pVCpu->iem.s.cbInstrBuf))
691 {
692 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
693 bRet = pbBuf[offBuf];
694 }
695 else
696 bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
697# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
698 Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
699 pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
700# endif
701 return bRet;
702
703# else /* !IEM_WITH_CODE_TLB */
704 uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
705 if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
706 {
707 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
708 return pVCpu->iem.s.abOpcode[offOpcode];
709 }
710 return iemOpcodeGetNextU8SlowJmp(pVCpu);
711# endif
712}
713
714# endif /* IEM_WITH_SETJMP */
715
716/**
717 * Fetches the next opcode byte, returns automatically on failure.
718 *
719 * @param a_pu8 Where to return the opcode byte.
720 * @remark Implicitly references pVCpu.
721 */
722# ifndef IEM_WITH_SETJMP
723# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
724 do \
725 { \
726 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
727 if (rcStrict2 == VINF_SUCCESS) \
728 { /* likely */ } \
729 else \
730 return rcStrict2; \
731 } while (0)
732# else
733# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
734# endif /* IEM_WITH_SETJMP */
735
736
737# ifndef IEM_WITH_SETJMP
738/**
739 * Fetches the next signed byte from the opcode stream.
740 *
741 * @returns Strict VBox status code.
742 * @param pVCpu The cross context virtual CPU structure of the calling thread.
743 * @param pi8 Where to return the signed byte.
744 */
745DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
746{
747 return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
748}
749# endif /* !IEM_WITH_SETJMP */
750
751
752/**
753 * Fetches the next signed byte from the opcode stream, returning automatically
754 * on failure.
755 *
756 * @param a_pi8 Where to return the signed byte.
757 * @remark Implicitly references pVCpu.
758 */
759# ifndef IEM_WITH_SETJMP
760# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
761 do \
762 { \
763 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
764 if (rcStrict2 != VINF_SUCCESS) \
765 return rcStrict2; \
766 } while (0)
767# else /* IEM_WITH_SETJMP */
768# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
769
770# endif /* IEM_WITH_SETJMP */
771
772
773# ifndef IEM_WITH_SETJMP
774/**
775 * Fetches the next signed byte from the opcode stream, extending it to
776 * unsigned 16-bit.
777 *
778 * @returns Strict VBox status code.
779 * @param pVCpu The cross context virtual CPU structure of the calling thread.
780 * @param pu16 Where to return the unsigned word.
781 */
782DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
783{
784 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
785 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
786 return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
787
788 *pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
789 pVCpu->iem.s.offOpcode = offOpcode + 1;
790 return VINF_SUCCESS;
791}
792# endif /* !IEM_WITH_SETJMP */
793
794/**
795 * Fetches the next signed byte from the opcode stream and sign-extending it to
796 * a word, returning automatically on failure.
797 *
798 * @param a_pu16 Where to return the word.
799 * @remark Implicitly references pVCpu.
800 */
801# ifndef IEM_WITH_SETJMP
802# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
803 do \
804 { \
805 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
806 if (rcStrict2 != VINF_SUCCESS) \
807 return rcStrict2; \
808 } while (0)
809# else
810# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
811# endif
812
813# ifndef IEM_WITH_SETJMP
814/**
815 * Fetches the next signed byte from the opcode stream, extending it to
816 * unsigned 32-bit.
817 *
818 * @returns Strict VBox status code.
819 * @param pVCpu The cross context virtual CPU structure of the calling thread.
820 * @param pu32 Where to return the unsigned dword.
821 */
822DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
823{
824 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
825 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
826 return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
827
828 *pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
829 pVCpu->iem.s.offOpcode = offOpcode + 1;
830 return VINF_SUCCESS;
831}
832# endif /* !IEM_WITH_SETJMP */
833
834/**
835 * Fetches the next signed byte from the opcode stream and sign-extending it to
836 * a word, returning automatically on failure.
837 *
838 * @param a_pu32 Where to return the word.
839 * @remark Implicitly references pVCpu.
840 */
841# ifndef IEM_WITH_SETJMP
842# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
843 do \
844 { \
845 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
846 if (rcStrict2 != VINF_SUCCESS) \
847 return rcStrict2; \
848 } while (0)
849# else
850# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
851# endif
852
853
854# ifndef IEM_WITH_SETJMP
855/**
856 * Fetches the next signed byte from the opcode stream, extending it to
857 * unsigned 64-bit.
858 *
859 * @returns Strict VBox status code.
860 * @param pVCpu The cross context virtual CPU structure of the calling thread.
861 * @param pu64 Where to return the unsigned qword.
862 */
863DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
864{
865 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
866 if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
867 return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
868
869 *pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
870 pVCpu->iem.s.offOpcode = offOpcode + 1;
871 return VINF_SUCCESS;
872}
873# endif /* !IEM_WITH_SETJMP */
874
875/**
876 * Fetches the next signed byte from the opcode stream and sign-extending it to
877 * a word, returning automatically on failure.
878 *
879 * @param a_pu64 Where to return the word.
880 * @remark Implicitly references pVCpu.
881 */
882# ifndef IEM_WITH_SETJMP
883# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
884 do \
885 { \
886 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
887 if (rcStrict2 != VINF_SUCCESS) \
888 return rcStrict2; \
889 } while (0)
890# else
891# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
892# endif
893
894
895# ifndef IEM_WITH_SETJMP
896
897/**
898 * Fetches the next opcode word.
899 *
900 * @returns Strict VBox status code.
901 * @param pVCpu The cross context virtual CPU structure of the calling thread.
902 * @param pu16 Where to return the opcode word.
903 */
904DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
905{
906 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
907 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
908 {
909 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
910# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
911 *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
912# else
913 *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
914# endif
915 return VINF_SUCCESS;
916 }
917 return iemOpcodeGetNextU16Slow(pVCpu, pu16);
918}
919
920# else /* IEM_WITH_SETJMP */
921
922/**
923 * Fetches the next opcode word, longjmp on error.
924 *
925 * @returns The opcode word.
926 * @param pVCpu The cross context virtual CPU structure of the calling thread.
927 */
928DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
929{
930# ifdef IEM_WITH_CODE_TLB
931 uint16_t u16Ret;
932 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
933 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
934 if (RT_LIKELY( pbBuf != NULL
935 && offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
936 {
937 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
938# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
939 u16Ret = *(uint16_t const *)&pbBuf[offBuf];
940# else
941 u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
942# endif
943 }
944 else
945 u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
946
947# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
948 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
949 Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
950# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
951 *(uint16_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
952# else
953 pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
954 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
955# endif
956 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
957# endif
958
959 return u16Ret;
960
961# else /* !IEM_WITH_CODE_TLB */
962 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
963 if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
964 {
965 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
966# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
967 return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
968# else
969 return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
970# endif
971 }
972 return iemOpcodeGetNextU16SlowJmp(pVCpu);
973# endif /* !IEM_WITH_CODE_TLB */
974}
975
976# endif /* IEM_WITH_SETJMP */
977
978/**
979 * Fetches the next opcode word, returns automatically on failure.
980 *
981 * @param a_pu16 Where to return the opcode word.
982 * @remark Implicitly references pVCpu.
983 */
984# ifndef IEM_WITH_SETJMP
985# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
986 do \
987 { \
988 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
989 if (rcStrict2 != VINF_SUCCESS) \
990 return rcStrict2; \
991 } while (0)
992# else
993# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
994# endif
995
996# ifndef IEM_WITH_SETJMP
997/**
998 * Fetches the next opcode word, zero extending it to a double word.
999 *
1000 * @returns Strict VBox status code.
1001 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1002 * @param pu32 Where to return the opcode double word.
1003 */
1004DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1005{
1006 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1007 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1008 return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1009
1010 *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1011 pVCpu->iem.s.offOpcode = offOpcode + 2;
1012 return VINF_SUCCESS;
1013}
1014# endif /* !IEM_WITH_SETJMP */
1015
1016/**
1017 * Fetches the next opcode word and zero extends it to a double word, returns
1018 * automatically on failure.
1019 *
1020 * @param a_pu32 Where to return the opcode double word.
1021 * @remark Implicitly references pVCpu.
1022 */
1023# ifndef IEM_WITH_SETJMP
1024# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1025 do \
1026 { \
1027 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1028 if (rcStrict2 != VINF_SUCCESS) \
1029 return rcStrict2; \
1030 } while (0)
1031# else
1032# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1033# endif
1034
1035# ifndef IEM_WITH_SETJMP
1036/**
1037 * Fetches the next opcode word, zero extending it to a quad word.
1038 *
1039 * @returns Strict VBox status code.
1040 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1041 * @param pu64 Where to return the opcode quad word.
1042 */
1043DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1044{
1045 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1046 if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1047 return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1048
1049 *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1050 pVCpu->iem.s.offOpcode = offOpcode + 2;
1051 return VINF_SUCCESS;
1052}
1053# endif /* !IEM_WITH_SETJMP */
1054
1055/**
1056 * Fetches the next opcode word and zero extends it to a quad word, returns
1057 * automatically on failure.
1058 *
1059 * @param a_pu64 Where to return the opcode quad word.
1060 * @remark Implicitly references pVCpu.
1061 */
1062# ifndef IEM_WITH_SETJMP
1063# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1064 do \
1065 { \
1066 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1067 if (rcStrict2 != VINF_SUCCESS) \
1068 return rcStrict2; \
1069 } while (0)
1070# else
1071# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1072# endif
1073
1074
1075# ifndef IEM_WITH_SETJMP
1076/**
1077 * Fetches the next signed word from the opcode stream.
1078 *
1079 * @returns Strict VBox status code.
1080 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1081 * @param pi16 Where to return the signed word.
1082 */
1083DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1084{
1085 return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1086}
1087# endif /* !IEM_WITH_SETJMP */
1088
1089
1090/**
1091 * Fetches the next signed word from the opcode stream, returning automatically
1092 * on failure.
1093 *
1094 * @param a_pi16 Where to return the signed word.
1095 * @remark Implicitly references pVCpu.
1096 */
1097# ifndef IEM_WITH_SETJMP
1098# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1099 do \
1100 { \
1101 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1102 if (rcStrict2 != VINF_SUCCESS) \
1103 return rcStrict2; \
1104 } while (0)
1105# else
1106# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1107# endif
1108
1109# ifndef IEM_WITH_SETJMP
1110
1111/**
1112 * Fetches the next opcode dword.
1113 *
1114 * @returns Strict VBox status code.
1115 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1116 * @param pu32 Where to return the opcode double word.
1117 */
1118DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1119{
1120 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1121 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1122 {
1123 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1124# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1125 *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1126# else
1127 *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1128 pVCpu->iem.s.abOpcode[offOpcode + 1],
1129 pVCpu->iem.s.abOpcode[offOpcode + 2],
1130 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1131# endif
1132 return VINF_SUCCESS;
1133 }
1134 return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1135}
1136
1137# else /* IEM_WITH_SETJMP */
1138
1139/**
1140 * Fetches the next opcode dword, longjmp on error.
1141 *
1142 * @returns The opcode dword.
1143 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1144 */
1145DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1146{
1147# ifdef IEM_WITH_CODE_TLB
1148 uint32_t u32Ret;
1149 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1150 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1151 if (RT_LIKELY( pbBuf != NULL
1152 && offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1153 {
1154 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1155# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1156 u32Ret = *(uint32_t const *)&pbBuf[offBuf];
1157# else
1158 u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1159 pbBuf[offBuf + 1],
1160 pbBuf[offBuf + 2],
1161 pbBuf[offBuf + 3]);
1162# endif
1163 }
1164 else
1165 u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1166
1167# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1168 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1169 Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1170# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1171 *(uint32_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1172# else
1173 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1174 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1175 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1176 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1177# endif
1178 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1179# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1180
1181 return u32Ret;
1182
1183# else /* !IEM_WITH_CODE_TLB */
1184 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1185 if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1186 {
1187 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1188# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1189 return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1190# else
1191 return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1192 pVCpu->iem.s.abOpcode[offOpcode + 1],
1193 pVCpu->iem.s.abOpcode[offOpcode + 2],
1194 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1195# endif
1196 }
1197 return iemOpcodeGetNextU32SlowJmp(pVCpu);
1198# endif
1199}
1200
1201# endif /* IEM_WITH_SETJMP */
1202
1203/**
1204 * Fetches the next opcode dword, returns automatically on failure.
1205 *
1206 * @param a_pu32 Where to return the opcode dword.
1207 * @remark Implicitly references pVCpu.
1208 */
1209# ifndef IEM_WITH_SETJMP
1210# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1211 do \
1212 { \
1213 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1214 if (rcStrict2 != VINF_SUCCESS) \
1215 return rcStrict2; \
1216 } while (0)
1217# else
1218# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1219# endif
1220
1221# ifndef IEM_WITH_SETJMP
1222/**
1223 * Fetches the next opcode dword, zero extending it to a quad word.
1224 *
1225 * @returns Strict VBox status code.
1226 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1227 * @param pu64 Where to return the opcode quad word.
1228 */
1229DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1230{
1231 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1232 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1233 return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1234
1235 *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1236 pVCpu->iem.s.abOpcode[offOpcode + 1],
1237 pVCpu->iem.s.abOpcode[offOpcode + 2],
1238 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1239 pVCpu->iem.s.offOpcode = offOpcode + 4;
1240 return VINF_SUCCESS;
1241}
1242# endif /* !IEM_WITH_SETJMP */
1243
1244/**
1245 * Fetches the next opcode dword and zero extends it to a quad word, returns
1246 * automatically on failure.
1247 *
1248 * @param a_pu64 Where to return the opcode quad word.
1249 * @remark Implicitly references pVCpu.
1250 */
1251# ifndef IEM_WITH_SETJMP
1252# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1253 do \
1254 { \
1255 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1256 if (rcStrict2 != VINF_SUCCESS) \
1257 return rcStrict2; \
1258 } while (0)
1259# else
1260# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1261# endif
1262
1263
1264# ifndef IEM_WITH_SETJMP
1265/**
1266 * Fetches the next signed double word from the opcode stream.
1267 *
1268 * @returns Strict VBox status code.
1269 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1270 * @param pi32 Where to return the signed double word.
1271 */
1272DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1273{
1274 return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1275}
1276# endif
1277
1278/**
1279 * Fetches the next signed double word from the opcode stream, returning
1280 * automatically on failure.
1281 *
1282 * @param a_pi32 Where to return the signed double word.
1283 * @remark Implicitly references pVCpu.
1284 */
1285# ifndef IEM_WITH_SETJMP
1286# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1287 do \
1288 { \
1289 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1290 if (rcStrict2 != VINF_SUCCESS) \
1291 return rcStrict2; \
1292 } while (0)
1293# else
1294# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1295# endif
1296
1297# ifndef IEM_WITH_SETJMP
1298/**
1299 * Fetches the next opcode dword, sign extending it into a quad word.
1300 *
1301 * @returns Strict VBox status code.
1302 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1303 * @param pu64 Where to return the opcode quad word.
1304 */
1305DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1306{
1307 uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1308 if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1309 return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1310
1311 int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1312 pVCpu->iem.s.abOpcode[offOpcode + 1],
1313 pVCpu->iem.s.abOpcode[offOpcode + 2],
1314 pVCpu->iem.s.abOpcode[offOpcode + 3]);
1315 *pu64 = (uint64_t)(int64_t)i32;
1316 pVCpu->iem.s.offOpcode = offOpcode + 4;
1317 return VINF_SUCCESS;
1318}
1319# endif /* !IEM_WITH_SETJMP */
1320
1321/**
1322 * Fetches the next opcode double word and sign extends it to a quad word,
1323 * returns automatically on failure.
1324 *
1325 * @param a_pu64 Where to return the opcode quad word.
1326 * @remark Implicitly references pVCpu.
1327 */
1328# ifndef IEM_WITH_SETJMP
1329# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1330 do \
1331 { \
1332 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1333 if (rcStrict2 != VINF_SUCCESS) \
1334 return rcStrict2; \
1335 } while (0)
1336# else
1337# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1338# endif
1339
1340# ifndef IEM_WITH_SETJMP
1341
1342/**
1343 * Fetches the next opcode qword.
1344 *
1345 * @returns Strict VBox status code.
1346 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1347 * @param pu64 Where to return the opcode qword.
1348 */
1349DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1350{
1351 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1352 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1353 {
1354# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1355 *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1356# else
1357 *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1358 pVCpu->iem.s.abOpcode[offOpcode + 1],
1359 pVCpu->iem.s.abOpcode[offOpcode + 2],
1360 pVCpu->iem.s.abOpcode[offOpcode + 3],
1361 pVCpu->iem.s.abOpcode[offOpcode + 4],
1362 pVCpu->iem.s.abOpcode[offOpcode + 5],
1363 pVCpu->iem.s.abOpcode[offOpcode + 6],
1364 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1365# endif
1366 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1367 return VINF_SUCCESS;
1368 }
1369 return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1370}
1371
1372# else /* IEM_WITH_SETJMP */
1373
1374/**
1375 * Fetches the next opcode qword, longjmp on error.
1376 *
1377 * @returns The opcode qword.
1378 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1379 */
1380DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1381{
1382# ifdef IEM_WITH_CODE_TLB
1383 uint64_t u64Ret;
1384 uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1385 uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1386 if (RT_LIKELY( pbBuf != NULL
1387 && offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1388 {
1389 pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1390# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1391 u64Ret = *(uint64_t const *)&pbBuf[offBuf];
1392# else
1393 u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1394 pbBuf[offBuf + 1],
1395 pbBuf[offBuf + 2],
1396 pbBuf[offBuf + 3],
1397 pbBuf[offBuf + 4],
1398 pbBuf[offBuf + 5],
1399 pbBuf[offBuf + 6],
1400 pbBuf[offBuf + 7]);
1401# endif
1402 }
1403 else
1404 u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1405
1406# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1407 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1408 Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1409# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1410 *(uint64_t *)&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1411# else
1412 pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1413 pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1414 pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1415 pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1416 pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1417 pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1418 pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1419 pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1420# endif
1421 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1422# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1423
1424 return u64Ret;
1425
1426# else /* !IEM_WITH_CODE_TLB */
1427 uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1428 if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1429 {
1430 pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1431# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1432 return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1433# else
1434 return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1435 pVCpu->iem.s.abOpcode[offOpcode + 1],
1436 pVCpu->iem.s.abOpcode[offOpcode + 2],
1437 pVCpu->iem.s.abOpcode[offOpcode + 3],
1438 pVCpu->iem.s.abOpcode[offOpcode + 4],
1439 pVCpu->iem.s.abOpcode[offOpcode + 5],
1440 pVCpu->iem.s.abOpcode[offOpcode + 6],
1441 pVCpu->iem.s.abOpcode[offOpcode + 7]);
1442# endif
1443 }
1444 return iemOpcodeGetNextU64SlowJmp(pVCpu);
1445# endif /* !IEM_WITH_CODE_TLB */
1446}
1447
1448# endif /* IEM_WITH_SETJMP */
1449
1450/**
1451 * Fetches the next opcode quad word, returns automatically on failure.
1452 *
1453 * @param a_pu64 Where to return the opcode quad word.
1454 * @remark Implicitly references pVCpu.
1455 */
1456# ifndef IEM_WITH_SETJMP
1457# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1458 do \
1459 { \
1460 VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1461 if (rcStrict2 != VINF_SUCCESS) \
1462 return rcStrict2; \
1463 } while (0)
1464# else
1465# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1466# endif
1467
1468/**
1469 * For fetching the opcode bytes for an ModR/M effective address, but throw
1470 * away the result.
1471 *
1472 * This is used when decoding undefined opcodes and such where we want to avoid
1473 * unnecessary MC blocks.
1474 *
1475 * @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1476 * used instead. At least for now...
1477 */
1478# ifndef IEM_WITH_SETJMP
1479# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1480 RTGCPTR GCPtrEff; \
1481 VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1482 if (rcStrict != VINF_SUCCESS) \
1483 return rcStrict; \
1484 } while (0)
1485# else
1486# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1487 (void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1488 } while (0)
1489# endif
1490
1491#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1492
1493
1494/** @name Misc Worker Functions.
1495 * @{
1496 */
1497
1498/**
1499 * Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1500 * not (kind of obsolete now).
1501 *
1502 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1503 */
1504#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1505
1506/**
1507 * Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1508 *
1509 * @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1510 * @param a_fEfl The new EFLAGS.
1511 */
1512#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1513
1514
1515/**
1516 * Loads a NULL data selector into a selector register, both the hidden and
1517 * visible parts, in protected mode.
1518 *
1519 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1520 * @param pSReg Pointer to the segment register.
1521 * @param uRpl The RPL.
1522 */
1523DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1524{
1525 /** @todo Testcase: write a testcase checking what happends when loading a NULL
1526 * data selector in protected mode. */
1527 pSReg->Sel = uRpl;
1528 pSReg->ValidSel = uRpl;
1529 pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1530 if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1531 {
1532 /* VT-x (Intel 3960x) observed doing something like this. */
1533 pSReg->Attr.u = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D | (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1534 pSReg->u32Limit = UINT32_MAX;
1535 pSReg->u64Base = 0;
1536 }
1537 else
1538 {
1539 pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1540 pSReg->u32Limit = 0;
1541 pSReg->u64Base = 0;
1542 }
1543}
1544
1545/** @} */
1546
1547
1548/*
1549 *
1550 * Helpers routines.
1551 * Helpers routines.
1552 * Helpers routines.
1553 *
1554 */
1555
1556#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1557
1558/**
1559 * Recalculates the effective operand size.
1560 *
1561 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1562 */
1563DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1564{
1565 switch (IEM_GET_CPU_MODE(pVCpu))
1566 {
1567 case IEMMODE_16BIT:
1568 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1569 break;
1570 case IEMMODE_32BIT:
1571 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1572 break;
1573 case IEMMODE_64BIT:
1574 switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP))
1575 {
1576 case 0:
1577 pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1578 break;
1579 case IEM_OP_PRF_SIZE_OP:
1580 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1581 break;
1582 case IEM_OP_PRF_SIZE_REX_W:
1583 case IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP:
1584 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1585 break;
1586 }
1587 break;
1588 default:
1589 AssertFailed();
1590 }
1591}
1592
1593
1594/**
1595 * Sets the default operand size to 64-bit and recalculates the effective
1596 * operand size.
1597 *
1598 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1599 */
1600DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1601{
1602 Assert(IEM_IS_64BIT_CODE(pVCpu));
1603 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1604 if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1605 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1606 else
1607 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1608}
1609
1610
1611/**
1612 * Sets the default operand size to 64-bit and recalculates the effective
1613 * operand size, with intel ignoring any operand size prefix (AMD respects it).
1614 *
1615 * This is for the relative jumps.
1616 *
1617 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1618 */
1619DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1620{
1621 Assert(IEM_IS_64BIT_CODE(pVCpu));
1622 pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1623 if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W | IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1624 || pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1625 pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1626 else
1627 pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1628}
1629
1630#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1631
1632
1633
1634/** @name Register Access.
1635 * @{
1636 */
1637
1638/**
1639 * Gets a reference (pointer) to the specified hidden segment register.
1640 *
1641 * @returns Hidden register reference.
1642 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1643 * @param iSegReg The segment register.
1644 */
1645DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1646{
1647 Assert(iSegReg < X86_SREG_COUNT);
1648 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1649 PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1650
1651 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1652 return pSReg;
1653}
1654
1655
1656/**
1657 * Ensures that the given hidden segment register is up to date.
1658 *
1659 * @returns Hidden register reference.
1660 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1661 * @param pSReg The segment register.
1662 */
1663DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1664{
1665 Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1666 NOREF(pVCpu);
1667 return pSReg;
1668}
1669
1670
1671/**
1672 * Gets a reference (pointer) to the specified segment register (the selector
1673 * value).
1674 *
1675 * @returns Pointer to the selector variable.
1676 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1677 * @param iSegReg The segment register.
1678 */
1679DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1680{
1681 Assert(iSegReg < X86_SREG_COUNT);
1682 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1683 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1684}
1685
1686
1687/**
1688 * Fetches the selector value of a segment register.
1689 *
1690 * @returns The selector value.
1691 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1692 * @param iSegReg The segment register.
1693 */
1694DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1695{
1696 Assert(iSegReg < X86_SREG_COUNT);
1697 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1698 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1699}
1700
1701
1702/**
1703 * Fetches the base address value of a segment register.
1704 *
1705 * @returns The selector value.
1706 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1707 * @param iSegReg The segment register.
1708 */
1709DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1710{
1711 Assert(iSegReg < X86_SREG_COUNT);
1712 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1713 return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1714}
1715
1716
1717/**
1718 * Gets a reference (pointer) to the specified general purpose register.
1719 *
1720 * @returns Register reference.
1721 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1722 * @param iReg The general purpose register.
1723 */
1724DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1725{
1726 Assert(iReg < 16);
1727 return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1728}
1729
1730
1731#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1732/**
1733 * Gets a reference (pointer) to the specified 8-bit general purpose register.
1734 *
1735 * Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1736 *
1737 * @returns Register reference.
1738 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1739 * @param iReg The register.
1740 */
1741DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1742{
1743 if (iReg < 4 || (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1744 {
1745 Assert(iReg < 16);
1746 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1747 }
1748 /* high 8-bit register. */
1749 Assert(iReg < 8);
1750 return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1751}
1752#endif
1753
1754
1755/**
1756 * Gets a reference (pointer) to the specified 8-bit general purpose register,
1757 * alternative version with extended (20) register index.
1758 *
1759 * @returns Register reference.
1760 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1761 * @param iRegEx The register. The 16 first are regular ones,
1762 * whereas 16 thru 19 maps to AH, CH, DH and BH.
1763 */
1764DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1765{
1766 /** @todo This could be done by double indexing on little endian hosts:
1767 * return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1768 if (iRegEx < 16)
1769 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1770
1771 /* high 8-bit register. */
1772 Assert(iRegEx < 20);
1773 return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1774}
1775
1776
1777/**
1778 * Gets a reference (pointer) to the specified 16-bit general purpose register.
1779 *
1780 * @returns Register reference.
1781 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1782 * @param iReg The register.
1783 */
1784DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1785{
1786 Assert(iReg < 16);
1787 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1788}
1789
1790
1791/**
1792 * Gets a reference (pointer) to the specified 32-bit general purpose register.
1793 *
1794 * @returns Register reference.
1795 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1796 * @param iReg The register.
1797 */
1798DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1799{
1800 Assert(iReg < 16);
1801 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1802}
1803
1804
1805/**
1806 * Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1807 *
1808 * @returns Register reference.
1809 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1810 * @param iReg The register.
1811 */
1812DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1813{
1814 Assert(iReg < 16);
1815 return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1816}
1817
1818
1819/**
1820 * Gets a reference (pointer) to the specified 64-bit general purpose register.
1821 *
1822 * @returns Register reference.
1823 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1824 * @param iReg The register.
1825 */
1826DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1827{
1828 Assert(iReg < 64);
1829 return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1830}
1831
1832
1833/**
1834 * Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1835 *
1836 * @returns Register reference.
1837 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1838 * @param iReg The register.
1839 */
1840DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1841{
1842 Assert(iReg < 16);
1843 return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1844}
1845
1846
1847/**
1848 * Gets a reference (pointer) to the specified segment register's base address.
1849 *
1850 * @returns Segment register base address reference.
1851 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1852 * @param iSegReg The segment selector.
1853 */
1854DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1855{
1856 Assert(iSegReg < X86_SREG_COUNT);
1857 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1858 return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1859}
1860
1861
1862#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1863/**
1864 * Fetches the value of a 8-bit general purpose register.
1865 *
1866 * @returns The register value.
1867 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1868 * @param iReg The register.
1869 */
1870DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1871{
1872 return *iemGRegRefU8(pVCpu, iReg);
1873}
1874#endif
1875
1876
1877/**
1878 * Fetches the value of a 8-bit general purpose register, alternative version
1879 * with extended (20) register index.
1880
1881 * @returns The register value.
1882 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1883 * @param iRegEx The register. The 16 first are regular ones,
1884 * whereas 16 thru 19 maps to AH, CH, DH and BH.
1885 */
1886DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1887{
1888 return *iemGRegRefU8Ex(pVCpu, iRegEx);
1889}
1890
1891
1892/**
1893 * Fetches the value of a 16-bit general purpose register.
1894 *
1895 * @returns The register value.
1896 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1897 * @param iReg The register.
1898 */
1899DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1900{
1901 Assert(iReg < 16);
1902 return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1903}
1904
1905
1906/**
1907 * Fetches the value of a 32-bit general purpose register.
1908 *
1909 * @returns The register value.
1910 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1911 * @param iReg The register.
1912 */
1913DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1914{
1915 Assert(iReg < 16);
1916 return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1917}
1918
1919
1920/**
1921 * Fetches the value of a 64-bit general purpose register.
1922 *
1923 * @returns The register value.
1924 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1925 * @param iReg The register.
1926 */
1927DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1928{
1929 Assert(iReg < 16);
1930 return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1931}
1932
1933
1934/**
1935 * Stores a 16-bit value to a general purpose register.
1936 *
1937 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1938 * @param iReg The register.
1939 * @param uValue The value to store.
1940 */
1941DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1942{
1943 Assert(iReg < 16);
1944 pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1945}
1946
1947
1948/**
1949 * Stores a 32-bit value to a general purpose register, implicitly clearing high
1950 * values.
1951 *
1952 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1953 * @param iReg The register.
1954 * @param uValue The value to store.
1955 */
1956DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1957{
1958 Assert(iReg < 16);
1959 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1960}
1961
1962
1963/**
1964 * Stores a 64-bit value to a general purpose register.
1965 *
1966 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1967 * @param iReg The register.
1968 * @param uValue The value to store.
1969 */
1970DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1971{
1972 Assert(iReg < 16);
1973 pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1974}
1975
1976
1977/**
1978 * Get the address of the top of the stack.
1979 *
1980 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1981 */
1982DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1983{
1984 if (IEM_IS_64BIT_CODE(pVCpu))
1985 return pVCpu->cpum.GstCtx.rsp;
1986 if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1987 return pVCpu->cpum.GstCtx.esp;
1988 return pVCpu->cpum.GstCtx.sp;
1989}
1990
1991
1992/**
1993 * Updates the RIP/EIP/IP to point to the next instruction.
1994 *
1995 * @param pVCpu The cross context virtual CPU structure of the calling thread.
1996 * @param cbInstr The number of bytes to add.
1997 */
1998DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1999{
2000 /*
2001 * Advance RIP.
2002 *
2003 * When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
2004 * while in all other modes except LM64 the updates are 32-bit. This means
2005 * we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
2006 * 4GB and 64KB rollovers, and decide whether anything needs masking.
2007 *
2008 * See PC wrap around tests in bs3-cpu-weird-1.
2009 */
2010 uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2011 uint64_t const uRipNext = uRipPrev + cbInstr;
2012 if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) | RT_BIT_64(16)))
2013 || IEM_IS_64BIT_CODE(pVCpu)))
2014 pVCpu->cpum.GstCtx.rip = uRipNext;
2015 else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2016 pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2017 else
2018 pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2019}
2020
2021
2022/**
2023 * Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2024 * following EFLAGS bits are set:
2025 * - X86_EFL_RF - clear it.
2026 * - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2027 * - X86_EFL_TF - generate single step \#DB trap.
2028 * - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2029 * instruction).
2030 *
2031 * According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2032 * a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2033 * takes priority over both NMIs and hardware interrupts. So, neither is
2034 * considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2035 * either unsupported will be triggered on-top of any \#DB raised here.)
2036 *
2037 * The RF flag only needs to be cleared here as it only suppresses instruction
2038 * breakpoints which are not raised here (happens synchronously during
2039 * instruction fetching).
2040 *
2041 * The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2042 * status has no bearing on whether \#DB exceptions are raised.
2043 *
2044 * @note This must *NOT* be called by the two instructions setting the
2045 * CPUMCTX_INHIBIT_SHADOW_SS flag.
2046 *
2047 * @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2048 * @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2049 * Stacks}
2050 */
2051static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2052{
2053 /*
2054 * Normally we're just here to clear RF and/or interrupt shadow bits.
2055 */
2056 if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) == 0))
2057 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);
2058 else
2059 {
2060 /*
2061 * Raise a #DB or/and DBGF event.
2062 */
2063 VBOXSTRICTRC rcStrict;
2064 if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF | CPUMCTX_DBG_HIT_DRX_MASK))
2065 {
2066 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2067 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2068 if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2069 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS;
2070 pVCpu->cpum.GstCtx.dr[6] |= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2071 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2072 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2073 pVCpu->cpum.GstCtx.rflags.uBoth));
2074
2075 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK);
2076 rcStrict = iemRaiseDebugException(pVCpu);
2077
2078 /* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2079 if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2080 {
2081 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2082 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2083 }
2084 }
2085 else
2086 {
2087 Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2088 rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2089 LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2090 }
2091 pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2092 return rcStrict;
2093 }
2094 return VINF_SUCCESS;
2095}
2096
2097
2098/**
2099 * Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2100 *
2101 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2102 */
2103DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2104{
2105 /*
2106 * We assume that most of the time nothing actually needs doing here.
2107 */
2108 AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2109 if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2110 & (X86_EFL_TF | X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK)) ))
2111 return VINF_SUCCESS;
2112 return iemFinishInstructionWithFlagsSet(pVCpu);
2113}
2114
2115
2116/**
2117 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2118 * and CPUMCTX_INHIBIT_SHADOW.
2119 *
2120 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2121 * @param cbInstr The number of bytes to add.
2122 */
2123DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2124{
2125 iemRegAddToRip(pVCpu, cbInstr);
2126 return iemRegFinishClearingRF(pVCpu);
2127}
2128
2129
2130/**
2131 * Updates the RIP to point to the next instruction and clears EFLAGS.RF
2132 * and CPUMCTX_INHIBIT_SHADOW.
2133 *
2134 * Only called from 64-bit code.
2135 *
2136 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2137 * @param cbInstr The number of bytes to add.
2138 */
2139DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2140{
2141 pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2142 return iemRegFinishClearingRF(pVCpu);
2143}
2144
2145
2146/**
2147 * Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2148 * CPUMCTX_INHIBIT_SHADOW.
2149 *
2150 * This is never from 64-bit code.
2151 *
2152 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2153 * @param cbInstr The number of bytes to add.
2154 */
2155DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2156{
2157 pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2158 return iemRegFinishClearingRF(pVCpu);
2159}
2160
2161
2162/**
2163 * Updates the IP to point to the next instruction and clears EFLAGS.RF and
2164 * CPUMCTX_INHIBIT_SHADOW.
2165 *
2166 * This is only ever used from 16-bit code on a pre-386 CPU.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2169 * @param cbInstr The number of bytes to add.
2170 */
2171DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2172{
2173 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2174 return iemRegFinishClearingRF(pVCpu);
2175}
2176
2177
2178/**
2179 * Adds a 8-bit signed jump offset to RIP from 64-bit code.
2180 *
2181 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2182 * segment limit.
2183 *
2184 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2185 * @param cbInstr Instruction size.
2186 * @param offNextInstr The offset of the next instruction.
2187 * @param enmEffOpSize Effective operand size.
2188 */
2189DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2190 IEMMODE enmEffOpSize) RT_NOEXCEPT
2191{
2192 Assert(IEM_IS_64BIT_CODE(pVCpu));
2193 Assert(enmEffOpSize == IEMMODE_64BIT || enmEffOpSize == IEMMODE_16BIT);
2194
2195 uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2196 if (enmEffOpSize == IEMMODE_16BIT)
2197 uNewRip &= UINT16_MAX;
2198
2199 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2200 pVCpu->cpum.GstCtx.rip = uNewRip;
2201 else
2202 return iemRaiseGeneralProtectionFault0(pVCpu);
2203
2204#ifndef IEM_WITH_CODE_TLB
2205 iemOpcodeFlushLight(pVCpu, cbInstr);
2206#endif
2207
2208 /*
2209 * Clear RF and finish the instruction (maybe raise #DB).
2210 */
2211 return iemRegFinishClearingRF(pVCpu);
2212}
2213
2214
2215/**
2216 * Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2217 * code (never 64-bit).
2218 *
2219 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2220 * segment limit.
2221 *
2222 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2223 * @param cbInstr Instruction size.
2224 * @param offNextInstr The offset of the next instruction.
2225 * @param enmEffOpSize Effective operand size.
2226 */
2227DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2228 IEMMODE enmEffOpSize) RT_NOEXCEPT
2229{
2230 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2231 Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
2232
2233 uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2234 if (enmEffOpSize == IEMMODE_16BIT)
2235 uNewEip &= UINT16_MAX;
2236 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2237 pVCpu->cpum.GstCtx.rip = uNewEip;
2238 else
2239 return iemRaiseGeneralProtectionFault0(pVCpu);
2240
2241#ifndef IEM_WITH_CODE_TLB
2242 iemOpcodeFlushLight(pVCpu, cbInstr);
2243#endif
2244
2245 /*
2246 * Clear RF and finish the instruction (maybe raise #DB).
2247 */
2248 return iemRegFinishClearingRF(pVCpu);
2249}
2250
2251
2252/**
2253 * Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2254 *
2255 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2256 * segment limit.
2257 *
2258 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2259 * @param cbInstr Instruction size.
2260 * @param offNextInstr The offset of the next instruction.
2261 */
2262DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2263 int8_t offNextInstr) RT_NOEXCEPT
2264{
2265 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2266
2267 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2268 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2269 pVCpu->cpum.GstCtx.rip = uNewIp;
2270 else
2271 return iemRaiseGeneralProtectionFault0(pVCpu);
2272
2273#ifndef IEM_WITH_CODE_TLB
2274 iemOpcodeFlushLight(pVCpu, cbInstr);
2275#endif
2276
2277 /*
2278 * Clear RF and finish the instruction (maybe raise #DB).
2279 */
2280 return iemRegFinishClearingRF(pVCpu);
2281}
2282
2283
2284/**
2285 * Adds a 16-bit signed jump offset to RIP from 64-bit code.
2286 *
2287 * @returns Strict VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2289 * @param cbInstr Instruction size.
2290 * @param offNextInstr The offset of the next instruction.
2291 */
2292DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2293 int16_t offNextInstr) RT_NOEXCEPT
2294{
2295 Assert(IEM_IS_64BIT_CODE(pVCpu));
2296
2297 pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2298
2299#ifndef IEM_WITH_CODE_TLB
2300 iemOpcodeFlushLight(pVCpu, cbInstr);
2301#endif
2302
2303 /*
2304 * Clear RF and finish the instruction (maybe raise #DB).
2305 */
2306 return iemRegFinishClearingRF(pVCpu);
2307}
2308
2309
2310/**
2311 * Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2312 *
2313 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2314 * segment limit.
2315 *
2316 * @returns Strict VBox status code.
2317 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2318 * @param cbInstr Instruction size.
2319 * @param offNextInstr The offset of the next instruction.
2320 *
2321 * @note This is also used by 16-bit code in pre-386 mode, as the code is
2322 * identical.
2323 */
2324DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2325 int16_t offNextInstr) RT_NOEXCEPT
2326{
2327 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2328
2329 uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2330 if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2331 pVCpu->cpum.GstCtx.rip = uNewIp;
2332 else
2333 return iemRaiseGeneralProtectionFault0(pVCpu);
2334
2335#ifndef IEM_WITH_CODE_TLB
2336 iemOpcodeFlushLight(pVCpu, cbInstr);
2337#endif
2338
2339 /*
2340 * Clear RF and finish the instruction (maybe raise #DB).
2341 */
2342 return iemRegFinishClearingRF(pVCpu);
2343}
2344
2345
2346/**
2347 * Adds a 32-bit signed jump offset to RIP from 64-bit code.
2348 *
2349 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2350 * segment limit.
2351 *
2352 * We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2353 * only alternative for relative jumps in 64-bit code and that is already
2354 * handled in the decoder stage.
2355 *
2356 * @returns Strict VBox status code.
2357 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2358 * @param cbInstr Instruction size.
2359 * @param offNextInstr The offset of the next instruction.
2360 */
2361DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2362 int32_t offNextInstr) RT_NOEXCEPT
2363{
2364 Assert(IEM_IS_64BIT_CODE(pVCpu));
2365
2366 uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2367 if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2368 pVCpu->cpum.GstCtx.rip = uNewRip;
2369 else
2370 return iemRaiseGeneralProtectionFault0(pVCpu);
2371
2372#ifndef IEM_WITH_CODE_TLB
2373 iemOpcodeFlushLight(pVCpu, cbInstr);
2374#endif
2375
2376 /*
2377 * Clear RF and finish the instruction (maybe raise #DB).
2378 */
2379 return iemRegFinishClearingRF(pVCpu);
2380}
2381
2382
2383/**
2384 * Adds a 32-bit signed jump offset to RIP from 64-bit code.
2385 *
2386 * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2387 * segment limit.
2388 *
2389 * We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2390 * only alternative for relative jumps in 32-bit code and that is already
2391 * handled in the decoder stage.
2392 *
2393 * @returns Strict VBox status code.
2394 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2395 * @param cbInstr Instruction size.
2396 * @param offNextInstr The offset of the next instruction.
2397 */
2398DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2399 int32_t offNextInstr) RT_NOEXCEPT
2400{
2401 Assert(!IEM_IS_64BIT_CODE(pVCpu));
2402 Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2403
2404 uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2405 if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2406 pVCpu->cpum.GstCtx.rip = uNewEip;
2407 else
2408 return iemRaiseGeneralProtectionFault0(pVCpu);
2409
2410#ifndef IEM_WITH_CODE_TLB
2411 iemOpcodeFlushLight(pVCpu, cbInstr);
2412#endif
2413
2414 /*
2415 * Clear RF and finish the instruction (maybe raise #DB).
2416 */
2417 return iemRegFinishClearingRF(pVCpu);
2418}
2419
2420
2421/**
2422 * Extended version of iemFinishInstructionWithFlagsSet that goes with
2423 * iemRegAddToRipAndFinishingClearingRfEx.
2424 *
2425 * See iemFinishInstructionWithFlagsSet() for details.
2426 */
2427static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2428{
2429 /*
2430 * Raise a #DB.
2431 */
2432 IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2433 pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2434 pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_BS
2435 | (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2436 /** @todo Do we set all pending \#DB events, or just one? */
2437 LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2438 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2439 pVCpu->cpum.GstCtx.rflags.uBoth));
2440 pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW | CPUMCTX_DBG_HIT_DRX_MASK | CPUMCTX_DBG_DBGF_MASK);
2441 return iemRaiseDebugException(pVCpu);
2442}
2443
2444
2445/**
2446 * Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2447 * others potentially updating EFLAGS.TF.
2448 *
2449 * The single step event must be generated using the TF value at the start of
2450 * the instruction, not the new value set by it.
2451 *
2452 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2453 * @param cbInstr The number of bytes to add.
2454 * @param fEflOld The EFLAGS at the start of the instruction
2455 * execution.
2456 */
2457DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2458{
2459 iemRegAddToRip(pVCpu, cbInstr);
2460 if (!(fEflOld & X86_EFL_TF))
2461 return iemRegFinishClearingRF(pVCpu);
2462 return iemFinishInstructionWithTfSet(pVCpu);
2463}
2464
2465
2466#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2467/**
2468 * Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2469 *
2470 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2471 */
2472DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2473{
2474 return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2475}
2476#endif
2477
2478
2479/**
2480 * Adds to the stack pointer.
2481 *
2482 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2483 * @param cbToAdd The number of bytes to add (8-bit!).
2484 */
2485DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2486{
2487 if (IEM_IS_64BIT_CODE(pVCpu))
2488 pVCpu->cpum.GstCtx.rsp += cbToAdd;
2489 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2490 pVCpu->cpum.GstCtx.esp += cbToAdd;
2491 else
2492 pVCpu->cpum.GstCtx.sp += cbToAdd;
2493}
2494
2495
2496/**
2497 * Subtracts from the stack pointer.
2498 *
2499 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2500 * @param cbToSub The number of bytes to subtract (8-bit!).
2501 */
2502DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2503{
2504 if (IEM_IS_64BIT_CODE(pVCpu))
2505 pVCpu->cpum.GstCtx.rsp -= cbToSub;
2506 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2507 pVCpu->cpum.GstCtx.esp -= cbToSub;
2508 else
2509 pVCpu->cpum.GstCtx.sp -= cbToSub;
2510}
2511
2512
2513/**
2514 * Adds to the temporary stack pointer.
2515 *
2516 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2517 * @param pTmpRsp The temporary SP/ESP/RSP to update.
2518 * @param cbToAdd The number of bytes to add (16-bit).
2519 */
2520DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2521{
2522 if (IEM_IS_64BIT_CODE(pVCpu))
2523 pTmpRsp->u += cbToAdd;
2524 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2525 pTmpRsp->DWords.dw0 += cbToAdd;
2526 else
2527 pTmpRsp->Words.w0 += cbToAdd;
2528}
2529
2530
2531/**
2532 * Subtracts from the temporary stack pointer.
2533 *
2534 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2535 * @param pTmpRsp The temporary SP/ESP/RSP to update.
2536 * @param cbToSub The number of bytes to subtract.
2537 * @remarks The @a cbToSub argument *MUST* be 16-bit, iemCImpl_enter is
2538 * expecting that.
2539 */
2540DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2541{
2542 if (IEM_IS_64BIT_CODE(pVCpu))
2543 pTmpRsp->u -= cbToSub;
2544 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2545 pTmpRsp->DWords.dw0 -= cbToSub;
2546 else
2547 pTmpRsp->Words.w0 -= cbToSub;
2548}
2549
2550
2551/**
2552 * Calculates the effective stack address for a push of the specified size as
2553 * well as the new RSP value (upper bits may be masked).
2554 *
2555 * @returns Effective stack addressf for the push.
2556 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2557 * @param cbItem The size of the stack item to pop.
2558 * @param puNewRsp Where to return the new RSP value.
2559 */
2560DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2561{
2562 RTUINT64U uTmpRsp;
2563 RTGCPTR GCPtrTop;
2564 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2565
2566 if (IEM_IS_64BIT_CODE(pVCpu))
2567 GCPtrTop = uTmpRsp.u -= cbItem;
2568 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2569 GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2570 else
2571 GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2572 *puNewRsp = uTmpRsp.u;
2573 return GCPtrTop;
2574}
2575
2576
2577/**
2578 * Gets the current stack pointer and calculates the value after a pop of the
2579 * specified size.
2580 *
2581 * @returns Current stack pointer.
2582 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2583 * @param cbItem The size of the stack item to pop.
2584 * @param puNewRsp Where to return the new RSP value.
2585 */
2586DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2587{
2588 RTUINT64U uTmpRsp;
2589 RTGCPTR GCPtrTop;
2590 uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2591
2592 if (IEM_IS_64BIT_CODE(pVCpu))
2593 {
2594 GCPtrTop = uTmpRsp.u;
2595 uTmpRsp.u += cbItem;
2596 }
2597 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2598 {
2599 GCPtrTop = uTmpRsp.DWords.dw0;
2600 uTmpRsp.DWords.dw0 += cbItem;
2601 }
2602 else
2603 {
2604 GCPtrTop = uTmpRsp.Words.w0;
2605 uTmpRsp.Words.w0 += cbItem;
2606 }
2607 *puNewRsp = uTmpRsp.u;
2608 return GCPtrTop;
2609}
2610
2611
2612/**
2613 * Calculates the effective stack address for a push of the specified size as
2614 * well as the new temporary RSP value (upper bits may be masked).
2615 *
2616 * @returns Effective stack addressf for the push.
2617 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2618 * @param pTmpRsp The temporary stack pointer. This is updated.
2619 * @param cbItem The size of the stack item to pop.
2620 */
2621DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2622{
2623 RTGCPTR GCPtrTop;
2624
2625 if (IEM_IS_64BIT_CODE(pVCpu))
2626 GCPtrTop = pTmpRsp->u -= cbItem;
2627 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2628 GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2629 else
2630 GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2631 return GCPtrTop;
2632}
2633
2634
2635/**
2636 * Gets the effective stack address for a pop of the specified size and
2637 * calculates and updates the temporary RSP.
2638 *
2639 * @returns Current stack pointer.
2640 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2641 * @param pTmpRsp The temporary stack pointer. This is updated.
2642 * @param cbItem The size of the stack item to pop.
2643 */
2644DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2645{
2646 RTGCPTR GCPtrTop;
2647 if (IEM_IS_64BIT_CODE(pVCpu))
2648 {
2649 GCPtrTop = pTmpRsp->u;
2650 pTmpRsp->u += cbItem;
2651 }
2652 else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2653 {
2654 GCPtrTop = pTmpRsp->DWords.dw0;
2655 pTmpRsp->DWords.dw0 += cbItem;
2656 }
2657 else
2658 {
2659 GCPtrTop = pTmpRsp->Words.w0;
2660 pTmpRsp->Words.w0 += cbItem;
2661 }
2662 return GCPtrTop;
2663}
2664
2665/** @} */
2666
2667
2668/** @name FPU access and helpers.
2669 *
2670 * @{
2671 */
2672
2673
2674/**
2675 * Hook for preparing to use the host FPU.
2676 *
2677 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2678 *
2679 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2680 */
2681DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2682{
2683#ifdef IN_RING3
2684 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2685#else
2686 CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2687#endif
2688 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2689}
2690
2691
2692/**
2693 * Hook for preparing to use the host FPU for SSE.
2694 *
2695 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2696 *
2697 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2698 */
2699DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2700{
2701 iemFpuPrepareUsage(pVCpu);
2702}
2703
2704
2705/**
2706 * Hook for preparing to use the host FPU for AVX.
2707 *
2708 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2709 *
2710 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2711 */
2712DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2713{
2714 iemFpuPrepareUsage(pVCpu);
2715}
2716
2717
2718/**
2719 * Hook for actualizing the guest FPU state before the interpreter reads it.
2720 *
2721 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2722 *
2723 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2724 */
2725DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2726{
2727#ifdef IN_RING3
2728 NOREF(pVCpu);
2729#else
2730 CPUMRZFpuStateActualizeForRead(pVCpu);
2731#endif
2732 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2733}
2734
2735
2736/**
2737 * Hook for actualizing the guest FPU state before the interpreter changes it.
2738 *
2739 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2740 *
2741 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2742 */
2743DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2744{
2745#ifdef IN_RING3
2746 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2747#else
2748 CPUMRZFpuStateActualizeForChange(pVCpu);
2749#endif
2750 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2751}
2752
2753
2754/**
2755 * Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2756 * only.
2757 *
2758 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2759 *
2760 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2761 */
2762DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2763{
2764#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
2765 NOREF(pVCpu);
2766#else
2767 CPUMRZFpuStateActualizeSseForRead(pVCpu);
2768#endif
2769 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2770}
2771
2772
2773/**
2774 * Hook for actualizing the guest XMM0..15 and MXCSR register state for
2775 * read+write.
2776 *
2777 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2778 *
2779 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2780 */
2781DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2782{
2783#if defined(IN_RING3) || defined(VBOX_WITH_KERNEL_USING_XMM)
2784 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2785#else
2786 CPUMRZFpuStateActualizeForChange(pVCpu);
2787#endif
2788 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2789
2790 /* Make sure any changes are loaded the next time around. */
2791 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_SSE;
2792}
2793
2794
2795/**
2796 * Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2797 * only.
2798 *
2799 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2800 *
2801 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2802 */
2803DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2804{
2805#ifdef IN_RING3
2806 NOREF(pVCpu);
2807#else
2808 CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2809#endif
2810 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2811}
2812
2813
2814/**
2815 * Hook for actualizing the guest YMM0..15 and MXCSR register state for
2816 * read+write.
2817 *
2818 * This is necessary in ring-0 and raw-mode context (nop in ring-3).
2819 *
2820 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2821 */
2822DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2823{
2824#ifdef IN_RING3
2825 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2826#else
2827 CPUMRZFpuStateActualizeForChange(pVCpu);
2828#endif
2829 IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
2830
2831 /* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2832 pVCpu->cpum.GstCtx.XState.Hdr.bmXState |= XSAVE_C_YMM | XSAVE_C_SSE;
2833}
2834
2835
2836/**
2837 * Stores a QNaN value into a FPU register.
2838 *
2839 * @param pReg Pointer to the register.
2840 */
2841DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2842{
2843 pReg->au32[0] = UINT32_C(0x00000000);
2844 pReg->au32[1] = UINT32_C(0xc0000000);
2845 pReg->au16[4] = UINT16_C(0xffff);
2846}
2847
2848
2849/**
2850 * Updates the FOP, FPU.CS and FPUIP registers, extended version.
2851 *
2852 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2853 * @param pFpuCtx The FPU context.
2854 * @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2855 */
2856DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2857{
2858 Assert(uFpuOpcode != UINT16_MAX);
2859 pFpuCtx->FOP = uFpuOpcode;
2860 /** @todo x87.CS and FPUIP needs to be kept seperately. */
2861 if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2862 {
2863 /** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2864 * happens in real mode here based on the fnsave and fnstenv images. */
2865 pFpuCtx->CS = 0;
2866 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip | ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2867 }
2868 else if (!IEM_IS_LONG_MODE(pVCpu))
2869 {
2870 pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2871 pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2872 }
2873 else
2874 *(uint64_t *)&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2875}
2876
2877
2878/**
2879 * Marks the specified stack register as free (for FFREE).
2880 *
2881 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2882 * @param iStReg The register to free.
2883 */
2884DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2885{
2886 Assert(iStReg < 8);
2887 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2888 uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2889 pFpuCtx->FTW &= ~RT_BIT(iReg);
2890}
2891
2892
2893/**
2894 * Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2895 *
2896 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2897 */
2898DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2899{
2900 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2901 uint16_t uFsw = pFpuCtx->FSW;
2902 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2903 uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2904 uFsw &= ~X86_FSW_TOP_MASK;
2905 uFsw |= uTop;
2906 pFpuCtx->FSW = uFsw;
2907}
2908
2909
2910/**
2911 * Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2912 *
2913 * @param pVCpu The cross context virtual CPU structure of the calling thread.
2914 */
2915DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2916{
2917 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2918 uint16_t uFsw = pFpuCtx->FSW;
2919 uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2920 uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2921 uFsw &= ~X86_FSW_TOP_MASK;
2922 uFsw |= uTop;
2923 pFpuCtx->FSW = uFsw;
2924}
2925
2926
2927
2928
2929DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2930{
2931 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2932 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2933 if (pFpuCtx->FTW & RT_BIT(iReg))
2934 return VINF_SUCCESS;
2935 return VERR_NOT_FOUND;
2936}
2937
2938
2939DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2940{
2941 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2942 uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2943 if (pFpuCtx->FTW & RT_BIT(iReg))
2944 {
2945 *ppRef = &pFpuCtx->aRegs[iStReg].r80;
2946 return VINF_SUCCESS;
2947 }
2948 return VERR_NOT_FOUND;
2949}
2950
2951
2952DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2953 uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2954{
2955 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2956 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2957 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2958 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2959 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
2960 {
2961 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2962 *ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2963 return VINF_SUCCESS;
2964 }
2965 return VERR_NOT_FOUND;
2966}
2967
2968
2969DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2970{
2971 PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2972 uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2973 uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2974 uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2975 if ((pFpuCtx->FTW & (RT_BIT(iReg0) | RT_BIT(iReg1))) == (RT_BIT(iReg0) | RT_BIT(iReg1)))
2976 {
2977 *ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2978 return VINF_SUCCESS;
2979 }
2980 return VERR_NOT_FOUND;
2981}
2982
2983
2984/**
2985 * Rotates the stack registers when setting new TOS.
2986 *
2987 * @param pFpuCtx The FPU context.
2988 * @param iNewTop New TOS value.
2989 * @remarks We only do this to speed up fxsave/fxrstor which
2990 * arrange the FP registers in stack order.
2991 * MUST be done before writing the new TOS (FSW).
2992 */
2993DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2994{
2995 uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2996 RTFLOAT80U ar80Temp[8];
2997
2998 if (iOldTop == iNewTop)
2999 return;
3000
3001 /* Unscrew the stack and get it into 'native' order. */
3002 ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
3003 ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
3004 ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
3005 ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
3006 ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
3007 ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
3008 ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
3009 ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3010
3011 /* Now rotate the stack to the new position. */
3012 pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3013 pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3014 pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3015 pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3016 pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3017 pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3018 pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3019 pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3020}
3021
3022
3023/**
3024 * Updates the FPU exception status after FCW is changed.
3025 *
3026 * @param pFpuCtx The FPU context.
3027 */
3028DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3029{
3030 uint16_t u16Fsw = pFpuCtx->FSW;
3031 if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3032 u16Fsw |= X86_FSW_ES | X86_FSW_B;
3033 else
3034 u16Fsw &= ~(X86_FSW_ES | X86_FSW_B);
3035 pFpuCtx->FSW = u16Fsw;
3036}
3037
3038
3039/**
3040 * Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3041 *
3042 * @returns The full FTW.
3043 * @param pFpuCtx The FPU context.
3044 */
3045DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3046{
3047 uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3048 uint16_t u16Ftw = 0;
3049 unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3050 for (unsigned iSt = 0; iSt < 8; iSt++)
3051 {
3052 unsigned const iReg = (iSt + iTop) & 7;
3053 if (!(u8Ftw & RT_BIT(iReg)))
3054 u16Ftw |= 3 << (iReg * 2); /* empty */
3055 else
3056 {
3057 uint16_t uTag;
3058 PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3059 if (pr80Reg->s.uExponent == 0x7fff)
3060 uTag = 2; /* Exponent is all 1's => Special. */
3061 else if (pr80Reg->s.uExponent == 0x0000)
3062 {
3063 if (pr80Reg->s.uMantissa == 0x0000)
3064 uTag = 1; /* All bits are zero => Zero. */
3065 else
3066 uTag = 2; /* Must be special. */
3067 }
3068 else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3069 uTag = 0; /* Valid. */
3070 else
3071 uTag = 2; /* Must be special. */
3072
3073 u16Ftw |= uTag << (iReg * 2);
3074 }
3075 }
3076
3077 return u16Ftw;
3078}
3079
3080
3081/**
3082 * Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3083 *
3084 * @returns The compressed FTW.
3085 * @param u16FullFtw The full FTW to convert.
3086 */
3087DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3088{
3089 uint8_t u8Ftw = 0;
3090 for (unsigned i = 0; i < 8; i++)
3091 {
3092 if ((u16FullFtw & 3) != 3 /*empty*/)
3093 u8Ftw |= RT_BIT(i);
3094 u16FullFtw >>= 2;
3095 }
3096
3097 return u8Ftw;
3098}
3099
3100/** @} */
3101
3102
3103/** @name Memory access.
3104 *
3105 * @{
3106 */
3107
3108
3109/**
3110 * Checks whether alignment checks are enabled or not.
3111 *
3112 * @returns true if enabled, false if not.
3113 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3114 */
3115DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3116{
3117 AssertCompile(X86_CR0_AM == X86_EFL_AC);
3118 return IEM_GET_CPL(pVCpu) == 3
3119 && (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3120}
3121
3122/**
3123 * Checks if the given segment can be written to, raise the appropriate
3124 * exception if not.
3125 *
3126 * @returns VBox strict status code.
3127 *
3128 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3129 * @param pHid Pointer to the hidden register.
3130 * @param iSegReg The register number.
3131 * @param pu64BaseAddr Where to return the base address to use for the
3132 * segment. (In 64-bit code it may differ from the
3133 * base in the hidden segment.)
3134 */
3135DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3136 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3137{
3138 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3139
3140 if (IEM_IS_64BIT_CODE(pVCpu))
3141 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3142 else
3143 {
3144 if (!pHid->Attr.n.u1Present)
3145 {
3146 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3147 AssertRelease(uSel == 0);
3148 Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3149 return iemRaiseGeneralProtectionFault0(pVCpu);
3150 }
3151
3152 if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3153 || !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3154 && !IEM_IS_64BIT_CODE(pVCpu) )
3155 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3156 *pu64BaseAddr = pHid->u64Base;
3157 }
3158 return VINF_SUCCESS;
3159}
3160
3161
3162/**
3163 * Checks if the given segment can be read from, raise the appropriate
3164 * exception if not.
3165 *
3166 * @returns VBox strict status code.
3167 *
3168 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3169 * @param pHid Pointer to the hidden register.
3170 * @param iSegReg The register number.
3171 * @param pu64BaseAddr Where to return the base address to use for the
3172 * segment. (In 64-bit code it may differ from the
3173 * base in the hidden segment.)
3174 */
3175DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3176 uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3177{
3178 IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3179
3180 if (IEM_IS_64BIT_CODE(pVCpu))
3181 *pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3182 else
3183 {
3184 if (!pHid->Attr.n.u1Present)
3185 {
3186 uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3187 AssertRelease(uSel == 0);
3188 Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3189 return iemRaiseGeneralProtectionFault0(pVCpu);
3190 }
3191
3192 if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3193 return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3194 *pu64BaseAddr = pHid->u64Base;
3195 }
3196 return VINF_SUCCESS;
3197}
3198
3199
3200/**
3201 * Maps a physical page.
3202 *
3203 * @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3204 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3205 * @param GCPhysMem The physical address.
3206 * @param fAccess The intended access.
3207 * @param ppvMem Where to return the mapping address.
3208 * @param pLock The PGM lock.
3209 */
3210DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3211 void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3212{
3213#ifdef IEM_LOG_MEMORY_WRITES
3214 if (fAccess & IEM_ACCESS_TYPE_WRITE)
3215 return VERR_PGM_PHYS_TLB_CATCH_ALL;
3216#endif
3217
3218 /** @todo This API may require some improving later. A private deal with PGM
3219 * regarding locking and unlocking needs to be struct. A couple of TLBs
3220 * living in PGM, but with publicly accessible inlined access methods
3221 * could perhaps be an even better solution. */
3222 int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3223 GCPhysMem,
3224 RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3225 RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3226 ppvMem,
3227 pLock);
3228 /*Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.*Rhxs\n", rc, sizeof(*pLock), pLock));*/
3229 AssertMsg(rc == VINF_SUCCESS || RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3230
3231 return rc;
3232}
3233
3234
3235/**
3236 * Unmap a page previously mapped by iemMemPageMap.
3237 *
3238 * @param pVCpu The cross context virtual CPU structure of the calling thread.
3239 * @param GCPhysMem The physical address.
3240 * @param fAccess The intended access.
3241 * @param pvMem What iemMemPageMap returned.
3242 * @param pLock The PGM lock.
3243 */
3244DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3245 const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3246{
3247 NOREF(pVCpu);
3248 NOREF(GCPhysMem);
3249 NOREF(fAccess);
3250 NOREF(pvMem);
3251 PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3252}
3253
3254#ifdef IEM_WITH_SETJMP
3255
3256/** @todo slim this down */
3257DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3258 size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3259{
3260 Assert(cbMem >= 1);
3261 Assert(iSegReg < X86_SREG_COUNT);
3262
3263 /*
3264 * 64-bit mode is simpler.
3265 */
3266 if (IEM_IS_64BIT_CODE(pVCpu))
3267 {
3268 if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3269 {
3270 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3271 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3272 GCPtrMem += pSel->u64Base;
3273 }
3274
3275 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3276 return GCPtrMem;
3277 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3278 }
3279 /*
3280 * 16-bit and 32-bit segmentation.
3281 */
3282 else if (iSegReg != UINT8_MAX)
3283 {
3284 /** @todo Does this apply to segments with 4G-1 limit? */
3285 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3286 if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3287 {
3288 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3289 PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3290 switch (pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
3291 | X86_SEL_TYPE_READ | X86_SEL_TYPE_WRITE /* same as read */
3292 | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_CONF /* same as down */
3293 | X86_SEL_TYPE_CODE))
3294 {
3295 case X86DESCATTR_P: /* readonly data, expand up */
3296 case X86DESCATTR_P | X86_SEL_TYPE_WRITE: /* writable data, expand up */
3297 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ: /* code, read-only */
3298 case X86DESCATTR_P | X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_CONF: /* conforming code, read-only */
3299 /* expand up */
3300 if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3301 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3302 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3303 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3304 break;
3305
3306 case X86DESCATTR_P | X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3307 case X86DESCATTR_P | X86_SEL_TYPE_DOWN | X86_SEL_TYPE_WRITE: /* writable data, expand down */
3308 /* expand down */
3309 if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3310 && ( pSel->Attr.n.u1DefBig
3311 || GCPtrLast32 <= UINT32_C(0xffff)) ))
3312 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3313 Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3314 (uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3315 break;
3316
3317 default:
3318 Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3319 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3320 break;
3321 }
3322 }
3323 Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3324 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3325 }
3326 /*
3327 * 32-bit flat address.
3328 */
3329 else
3330 return GCPtrMem;
3331}
3332
3333
3334/** @todo slim this down */
3335DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3336 RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3337{
3338 Assert(cbMem >= 1);
3339 Assert(iSegReg < X86_SREG_COUNT);
3340
3341 /*
3342 * 64-bit mode is simpler.
3343 */
3344 if (IEM_IS_64BIT_CODE(pVCpu))
3345 {
3346 if (iSegReg >= X86_SREG_FS)
3347 {
3348 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3349 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3350 GCPtrMem += pSel->u64Base;
3351 }
3352
3353 if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3354 return GCPtrMem;
3355 }
3356 /*
3357 * 16-bit and 32-bit segmentation.
3358 */
3359 else
3360 {
3361 Assert(GCPtrMem <= UINT32_MAX);
3362 IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3363 PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3364 uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P | X86DESCATTR_UNUSABLE
3365 | X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN);
3366 if ( fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE) /* data, expand up */
3367 /** @todo explore exactly how the CS stuff works in real mode. See also
3368 * http://www.rcollins.org/Productivity/DescriptorCache.html and
3369 * http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */
3370 || (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. */ /** @todo testcase! */
3371 {
3372 /* expand up */
3373 uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3374 if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit
3375 && GCPtrLast32 >= (uint32_t)GCPtrMem))
3376 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3377 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3378 }
3379 else if (fRelevantAttrs == (X86DESCATTR_P | X86_SEL_TYPE_WRITE | X86_SEL_TYPE_DOWN)) /* data, expand up */
3380 {
3381 /* expand down - the uppger boundary is defined by the B bit, not G. */
3382 uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3383 if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit
3384 && (pSel->Attr.n.u1DefBig || GCPtrLast32 <= UINT32_C(0xffff))
3385 && GCPtrLast32 >= (uint32_t)GCPtrMem))
3386 return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3387 iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3388 }
3389 else
3390 iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3391 }
3392 iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3393}
3394
3395#endif /* IEM_WITH_SETJMP */
3396
3397/**
3398 * Fakes a long mode stack selector for SS = 0.
3399 *
3400 * @param pDescSs Where to return the fake stack descriptor.
3401 * @param uDpl The DPL we want.
3402 */
3403DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3404{
3405 pDescSs->Long.au64[0] = 0;
3406 pDescSs->Long.au64[1] = 0;
3407 pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3408 pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3409 pDescSs->Long.Gen.u2Dpl = uDpl;
3410 pDescSs->Long.Gen.u1Present = 1;
3411 pDescSs->Long.Gen.u1Long = 1;
3412}
3413
3414
3415/*
3416 * Unmap helpers.
3417 */
3418
3419#ifdef IEM_WITH_SETJMP
3420
3421DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3422{
3423# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3424 if (RT_LIKELY(bMapInfo == 0))
3425 return;
3426# endif
3427 iemMemCommitAndUnmapRwSafeJmp(pVCpu, pvMem, bMapInfo);
3428}
3429
3430
3431DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3432{
3433# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3434 if (RT_LIKELY(bMapInfo == 0))
3435 return;
3436# endif
3437 iemMemCommitAndUnmapWoSafeJmp(pVCpu, pvMem, bMapInfo);
3438}
3439
3440
3441DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, const void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3442{
3443# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3444 if (RT_LIKELY(bMapInfo == 0))
3445 return;
3446# endif
3447 iemMemCommitAndUnmapRoSafeJmp(pVCpu, pvMem, bMapInfo);
3448}
3449
3450#endif /* IEM_WITH_SETJMP */
3451
3452
3453/*
3454 * Instantiate R/W inline templates.
3455 */
3456
3457/** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3458 * Used to check if an unaligned access is if within the page and won't
3459 * trigger an #AC.
3460 *
3461 * This can be used to deal with misaligned accesses on platforms that are
3462 * senstive to such if desires.
3463 */
3464AssertCompile(X86_CR0_AM == X86_EFL_AC);
3465AssertCompile(((3U + 1U) << 16) == X86_CR0_AM);
3466#if 1
3467# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \
3468 ( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \
3469 && !( (uint32_t)(a_pVCpu)->cpum.GstCtx.cr0 \
3470 & (a_pVCpu)->cpum.GstCtx.eflags.u \
3471 & ((IEM_GET_CPL((a_pVCpu)) + 1U) << 16) /* IEM_GET_CPL(a_pVCpu) == 3 ? X86_CR0_AM : 0 */ \
3472 & X86_CR0_AM) )
3473#else
3474# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0
3475#endif
3476
3477
3478#define TMPL_MEM_TYPE uint8_t
3479#define TMPL_MEM_TYPE_ALIGN 0
3480#define TMPL_MEM_TYPE_SIZE 1
3481#define TMPL_MEM_FN_SUFF U8
3482#define TMPL_MEM_FMT_TYPE "%#04x"
3483#define TMPL_MEM_FMT_DESC "byte"
3484#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3485
3486#define TMPL_MEM_TYPE uint16_t
3487#define TMPL_MEM_TYPE_ALIGN 1
3488#define TMPL_MEM_TYPE_SIZE 2
3489#define TMPL_MEM_FN_SUFF U16
3490#define TMPL_MEM_FMT_TYPE "%#06x"
3491#define TMPL_MEM_FMT_DESC "word"
3492#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3493
3494#define TMPL_MEM_TYPE uint32_t
3495#define TMPL_MEM_TYPE_ALIGN 3
3496#define TMPL_MEM_TYPE_SIZE 4
3497#define TMPL_MEM_FN_SUFF U32
3498#define TMPL_MEM_FMT_TYPE "%#010x"
3499#define TMPL_MEM_FMT_DESC "dword"
3500#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3501
3502#define TMPL_MEM_TYPE uint64_t
3503#define TMPL_MEM_TYPE_ALIGN 7
3504#define TMPL_MEM_TYPE_SIZE 8
3505#define TMPL_MEM_FN_SUFF U64
3506#define TMPL_MEM_FMT_TYPE "%#018RX64"
3507#define TMPL_MEM_FMT_DESC "qword"
3508#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3509
3510#define TMPL_MEM_NO_STORE
3511#define TMPL_MEM_NO_MAPPING
3512#define TMPL_MEM_TYPE uint64_t
3513#define TMPL_MEM_TYPE_ALIGN 15
3514#define TMPL_MEM_TYPE_SIZE 8
3515#define TMPL_MEM_FN_SUFF U64AlignedU128
3516#define TMPL_MEM_FMT_TYPE "%#018RX64"
3517#define TMPL_MEM_FMT_DESC "qword"
3518#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3519
3520#undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3521/** @} */
3522
3523
3524#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3525
3526/**
3527 * Gets CR0 fixed-0 bits in VMX operation.
3528 *
3529 * We do this rather than fetching what we report to the guest (in
3530 * IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3531 * values regardless of whether unrestricted-guest feature is available on the CPU.
3532 *
3533 * @returns CR0 fixed-0 bits.
3534 * @param pVCpu The cross context virtual CPU structure.
3535 * @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3536 * must be returned. When @c false, the CR0 fixed-0
3537 * bits for VMX root mode is returned.
3538 *
3539 */
3540DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3541{
3542 Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3543
3544 PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3545 if ( fVmxNonRootMode
3546 && (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3547 return VMX_V_CR0_FIXED0_UX;
3548 return VMX_V_CR0_FIXED0;
3549}
3550
3551
3552/**
3553 * Sets virtual-APIC write emulation as pending.
3554 *
3555 * @param pVCpu The cross context virtual CPU structure.
3556 * @param offApic The offset in the virtual-APIC page that was written.
3557 */
3558DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3559{
3560 Assert(offApic < XAPIC_OFF_END + 4);
3561
3562 /*
3563 * Record the currently updated APIC offset, as we need this later for figuring
3564 * out whether to perform TPR, EOI or self-IPI virtualization as well as well
3565 * as for supplying the exit qualification when causing an APIC-write VM-exit.
3566 */
3567 pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3568
3569 /*
3570 * Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3571 * virtualization or APIC-write emulation).
3572 */
3573 if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3574 VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3575}
3576
3577#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3578
3579#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette