IEMInline.h@ 100889

Last change on this file since 100889 was 100889, checked in by vboxsync, 16 months ago
VMM/IEM: Adjusted IEM_MC_CALL_CIMPL_HLP_RET assertion to account for fExec not including the FLAT 32-bit flag if DS, ES or SS state is stored externally (HM, NEM). bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 124.4 KB

Line
1	/* $Id: IEMInline.h 100889 2023-08-16 22:31:44Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	/* Deal with the cases that should be treated as VINF_SUCCESS first. */
49	if ( rcStrict == VINF_IEM_YIELD_PENDING_FF
50	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */
51	\|\| rcStrict == VINF_VMX_VMEXIT
52	#endif
53	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
54	\|\| rcStrict == VINF_SVM_VMEXIT
55	#endif
56	)
57	{
58	if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
59	rcStrict = VINF_SUCCESS;
60	else
61	{
62	pVCpu->iem.s.cRetPassUpStatus++;
63	rcStrict = pVCpu->iem.s.rcPassUp;
64	}
65	}
66	else if (RT_SUCCESS(rcStrict))
67	{
68	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
69	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
70	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
71	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
72	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
73	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
74	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
75	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
76	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
77	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
78	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
79	\|\| rcStrict == VINF_EM_RAW_TO_R3
80	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
81	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
82	/* raw-mode / virt handlers only: */
83	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
84	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
85	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
86	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
87	\|\| rcStrict == VINF_SELM_SYNC_GDT
88	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
89	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
90	/* nested hw.virt codes: */
91	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
92	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
93	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
94	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
95	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
96	if (rcPassUp == VINF_SUCCESS)
97	pVCpu->iem.s.cRetInfStatuses++;
98	else if ( rcPassUp < VINF_EM_FIRST
99	\|\| rcPassUp > VINF_EM_LAST
100	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
101	{
102	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
103	pVCpu->iem.s.cRetPassUpStatus++;
104	rcStrict = rcPassUp;
105	}
106	else
107	{
108	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
109	pVCpu->iem.s.cRetInfStatuses++;
110	}
111	}
112	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
113	pVCpu->iem.s.cRetAspectNotImplemented++;
114	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
115	pVCpu->iem.s.cRetInstrNotImplemented++;
116	else
117	pVCpu->iem.s.cRetErrStatuses++;
118	}
119	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
120	{
121	pVCpu->iem.s.cRetPassUpStatus++;
122	rcStrict = pVCpu->iem.s.rcPassUp;
123	}
124
125	/* Just clear it here as well. */
126	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
127
128	return rcStrict;
129	}
130
131
132	/**
133	* Sets the pass up status.
134	*
135	* @returns VINF_SUCCESS.
136	* @param pVCpu The cross context virtual CPU structure of the
137	* calling thread.
138	* @param rcPassUp The pass up status. Must be informational.
139	* VINF_SUCCESS is not allowed.
140	*/
141	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
142	{
143	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
144
145	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
146	if (rcOldPassUp == VINF_SUCCESS)
147	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148	/* If both are EM scheduling codes, use EM priority rules. */
149	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
150	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
151	{
152	if (rcPassUp < rcOldPassUp)
153	{
154	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
155	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
156	}
157	else
158	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
159	}
160	/* Override EM scheduling with specific status code. */
161	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
162	{
163	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
164	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
165	}
166	/* Don't override specific status code, first come first served. */
167	else
168	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
169	return VINF_SUCCESS;
170	}
171
172
173	/**
174	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
175	*
176	* Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
177	* reject expand down data segments and conforming code segments.
178	*
179	* ASSUMES that the CPU is in 32-bit mode.
180	*
181	* @note Will return zero when if any of the segment register state is marked
182	* external, this must be factored into assertions checking fExec
183	* consistency.
184	*
185	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
186	* @param pVCpu The cross context virtual CPU structure of the
187	* calling thread.
188	* @sa iemCalc32BitFlatIndicatorEsDs
189	*/
190	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
191	{
192	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
193	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
194	\| pVCpu->cpum.GstCtx.cs.Attr.u
195	\| pVCpu->cpum.GstCtx.ss.Attr.u
196	\| pVCpu->cpum.GstCtx.ds.Attr.u)
197	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
198	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
199	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
200	\| (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
201	\| (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
202	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
203	== 0
204	&& ( pVCpu->cpum.GstCtx.es.u64Base
205	\| pVCpu->cpum.GstCtx.cs.u64Base
206	\| pVCpu->cpum.GstCtx.ss.u64Base
207	\| pVCpu->cpum.GstCtx.ds.u64Base)
208	== 0
209	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
210	? IEM_F_MODE_X86_32BIT_FLAT : 0;
211	}
212
213
214	/**
215	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
216	* flat already.
217	*
218	* This is used by sysenter.
219	*
220	* @note Will return zero when if any of the segment register state is marked
221	* external, this must be factored into assertions checking fExec
222	* consistency.
223	*
224	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
225	* @param pVCpu The cross context virtual CPU structure of the
226	* calling thread.
227	* @sa iemCalc32BitFlatIndicator
228	*/
229	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
230	{
231	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
232	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
233	\| pVCpu->cpum.GstCtx.ds.Attr.u)
234	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
235	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
236	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
237	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
238	== 0
239	&& ( pVCpu->cpum.GstCtx.es.u64Base
240	\| pVCpu->cpum.GstCtx.ds.u64Base)
241	== 0
242	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
243	? IEM_F_MODE_X86_32BIT_FLAT : 0;
244	}
245
246
247	/**
248	* Calculates the IEM_F_MODE_XXX and CPL flags.
249	*
250	* @returns IEM_F_MODE_XXX
251	* @param pVCpu The cross context virtual CPU structure of the
252	* calling thread.
253	*/
254	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
255	{
256	/*
257	* We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
258	* here to try get this done as efficiently as possible.
259	*/
260	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
261
262	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
263	{
264	if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
265	{
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
268	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
269	{
270	Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
271	fExec \|= IEM_F_MODE_X86_32BIT_PROT \| iemCalc32BitFlatIndicator(pVCpu);
272	}
273	else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
274	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
275	fExec \|= IEM_F_MODE_X86_64BIT;
276	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
277	fExec \|= IEM_F_MODE_X86_16BIT_PROT;
278	else
279	fExec \|= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
280	return fExec;
281	}
282	return IEM_F_MODE_X86_16BIT_PROT_V86 \| (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
283	}
284
285	/* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
286	if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
287	{
288	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
289	return IEM_F_MODE_X86_16BIT;
290	return IEM_F_MODE_X86_16BIT_PRE_386;
291	}
292
293	/* 32-bit unreal mode. */
294	return IEM_F_MODE_X86_32BIT \| iemCalc32BitFlatIndicator(pVCpu);
295	}
296
297
298	/**
299	* Calculates the AMD-V and VT-x related context flags.
300	*
301	* @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
302	* IEM_F_X86_CTX_VMX.
303	* @param pVCpu The cross context virtual CPU structure of the
304	* calling thread.
305	*/
306	DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
307	{
308	/*
309	* This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
310	* and CPUMIsGuestInNestedHwvirtMode to some extent.
311	*/
312	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
313
314	AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
315	uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
316	\| (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
317	if (RT_LIKELY(!fTmp))
318	return 0; /* likely */
319
320	if (fTmp & X86_CR4_VMXE)
321	{
322	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
323	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
324	return IEM_F_X86_CTX_VMX \| IEM_F_X86_CTX_IN_GUEST;
325	return IEM_F_X86_CTX_VMX;
326	}
327
328	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
329	if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
330	return IEM_F_X86_CTX_SVM \| IEM_F_X86_CTX_IN_GUEST;
331	return IEM_F_X86_CTX_SVM;
332	}
333
334
335	/**
336	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
337	*
338	* @returns IEM_F_BRK_PENDING_XXX or zero.
339	* @param pVCpu The cross context virtual CPU structure of the
340	* calling thread.
341	*/
342	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
343	{
344	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
345
346	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
347	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
348	return 0;
349	return iemCalcExecDbgFlagsSlow(pVCpu);
350	}
351
352	/**
353	* Calculates the the IEM_F_XXX flags.
354	*
355	* @returns IEM_F_XXX combination match the current CPU state.
356	* @param pVCpu The cross context virtual CPU structure of the
357	* calling thread.
358	*/
359	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
360	{
361	return iemCalcExecModeAndCplFlags(pVCpu)
362	\| iemCalcExecHwVirtFlags(pVCpu)
363	/* SMM is not yet implemented */
364	\| iemCalcExecDbgFlags(pVCpu)
365	;
366	}
367
368
369	/**
370	* Re-calculates the MODE and CPL parts of IEMCPU::fExec.
371	*
372	* @param pVCpu The cross context virtual CPU structure of the
373	* calling thread.
374	*/
375	DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
376	{
377	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
378	\| iemCalcExecModeAndCplFlags(pVCpu);
379	}
380
381
382	/**
383	* Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
384	*
385	* @param pVCpu The cross context virtual CPU structure of the
386	* calling thread.
387	*/
388	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
389	{
390	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
391	\| iemCalcExecDbgFlags(pVCpu);
392	}
393
394
395	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
396
397	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
398	/**
399	* Initializes the execution state.
400	*
401	* @param pVCpu The cross context virtual CPU structure of the
402	* calling thread.
403	* @param fExecOpts Optional execution flags:
404	* - IEM_F_BYPASS_HANDLERS
405	* - IEM_F_X86_DISREGARD_LOCK
406	*
407	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
408	* side-effects in strict builds.
409	*/
410	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
411	{
412	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
413	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
414	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
415	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
416	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
417	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
418	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
419	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
420	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
421	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
422
423	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
424	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
425	pVCpu->iem.s.cActiveMappings = 0;
426	pVCpu->iem.s.iNextMapping = 0;
427
428	# ifdef VBOX_STRICT
429	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
430	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
431	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
432	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
433	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
434	pVCpu->iem.s.uRexReg = 127;
435	pVCpu->iem.s.uRexB = 127;
436	pVCpu->iem.s.offModRm = 127;
437	pVCpu->iem.s.uRexIndex = 127;
438	pVCpu->iem.s.iEffSeg = 127;
439	pVCpu->iem.s.idxPrefix = 127;
440	pVCpu->iem.s.uVex3rdReg = 127;
441	pVCpu->iem.s.uVexLength = 127;
442	pVCpu->iem.s.fEvexStuff = 127;
443	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
444	# ifdef IEM_WITH_CODE_TLB
445	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
446	pVCpu->iem.s.pbInstrBuf = NULL;
447	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
448	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
449	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
450	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
451	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
452	pVCpu->iem.s.offOpcode = 127;
453	# endif
454	# else
455	pVCpu->iem.s.offOpcode = 127;
456	pVCpu->iem.s.cbOpcode = 127;
457	# endif
458	# endif /* VBOX_STRICT */
459	}
460	# endif /* VBOX_INCLUDED_vmm_dbgf_h */
461
462
463	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
464	/**
465	* Performs a minimal reinitialization of the execution state.
466	*
467	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
468	* 'world-switch' types operations on the CPU. Currently only nested
469	* hardware-virtualization uses it.
470	*
471	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
472	* @param cbInstr The instruction length (for flushing).
473	*/
474	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
475	{
476	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
477	iemOpcodeFlushHeavy(pVCpu, cbInstr);
478	}
479	# endif
480
481
482	/**
483	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
484	*
485	* @param pVCpu The cross context virtual CPU structure of the
486	* calling thread.
487	*/
488	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
489	{
490	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
491	# ifdef VBOX_STRICT
492	# ifdef IEM_WITH_CODE_TLB
493	NOREF(pVCpu);
494	# else
495	pVCpu->iem.s.cbOpcode = 0;
496	# endif
497	# else
498	NOREF(pVCpu);
499	# endif
500	}
501
502
503	/**
504	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
505	*
506	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
507	*
508	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
509	* @param pVCpu The cross context virtual CPU structure of the calling thread.
510	* @param rcStrict The status code to fiddle.
511	*/
512	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
513	{
514	iemUninitExec(pVCpu);
515	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
516	}
517
518
519	/**
520	* Macro used by the IEMExec* method to check the given instruction length.
521	*
522	* Will return on failure!
523	*
524	* @param a_cbInstr The given instruction length.
525	* @param a_cbMin The minimum length.
526	*/
527	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
528	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
529	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
530
531
532	# ifndef IEM_WITH_SETJMP
533
534	/**
535	* Fetches the first opcode byte.
536	*
537	* @returns Strict VBox status code.
538	* @param pVCpu The cross context virtual CPU structure of the
539	* calling thread.
540	* @param pu8 Where to return the opcode byte.
541	*/
542	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
543	{
544	/*
545	* Check for hardware instruction breakpoints.
546	*/
547	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
548	{ /* likely */ }
549	else
550	{
551	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
552	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
553	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
554	{ /* likely */ }
555	else
556	{
557	pu8 = 0xff; / shut up gcc. sigh */
558	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
559	return iemRaiseDebugException(pVCpu);
560	return rcStrict;
561	}
562	}
563
564	/*
565	* Fetch the first opcode byte.
566	*/
567	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
568	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
569	{
570	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
571	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
572	return VINF_SUCCESS;
573	}
574	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
575	}
576
577	# else /* IEM_WITH_SETJMP */
578
579	/**
580	* Fetches the first opcode byte, longjmp on error.
581	*
582	* @returns The opcode byte.
583	* @param pVCpu The cross context virtual CPU structure of the calling thread.
584	*/
585	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
586	{
587	/*
588	* Check for hardware instruction breakpoints.
589	*/
590	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
591	{ /* likely */ }
592	else
593	{
594	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
595	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
596	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
597	{ /* likely */ }
598	else
599	{
600	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
601	rcStrict = iemRaiseDebugException(pVCpu);
602	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
603	}
604	}
605
606	/*
607	* Fetch the first opcode byte.
608	*/
609	# ifdef IEM_WITH_CODE_TLB
610	uint8_t bRet;
611	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
612	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
613	if (RT_LIKELY( pbBuf != NULL
614	&& offBuf < pVCpu->iem.s.cbInstrBuf))
615	{
616	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
617	bRet = pbBuf[offBuf];
618	}
619	else
620	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
621	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
622	Assert(pVCpu->iem.s.offOpcode == 0);
623	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
624	# endif
625	return bRet;
626
627	# else /* !IEM_WITH_CODE_TLB */
628	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
629	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
630	{
631	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
632	return pVCpu->iem.s.abOpcode[offOpcode];
633	}
634	return iemOpcodeGetNextU8SlowJmp(pVCpu);
635	# endif
636	}
637
638	# endif /* IEM_WITH_SETJMP */
639
640	/**
641	* Fetches the first opcode byte, returns/throws automatically on failure.
642	*
643	* @param a_pu8 Where to return the opcode byte.
644	* @remark Implicitly references pVCpu.
645	*/
646	# ifndef IEM_WITH_SETJMP
647	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
648	do \
649	{ \
650	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
651	if (rcStrict2 == VINF_SUCCESS) \
652	{ /* likely */ } \
653	else \
654	return rcStrict2; \
655	} while (0)
656	# else
657	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
658	# endif /* IEM_WITH_SETJMP */
659
660
661	# ifndef IEM_WITH_SETJMP
662
663	/**
664	* Fetches the next opcode byte.
665	*
666	* @returns Strict VBox status code.
667	* @param pVCpu The cross context virtual CPU structure of the
668	* calling thread.
669	* @param pu8 Where to return the opcode byte.
670	*/
671	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
672	{
673	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
674	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
675	{
676	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
677	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
678	return VINF_SUCCESS;
679	}
680	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
681	}
682
683	# else /* IEM_WITH_SETJMP */
684
685	/**
686	* Fetches the next opcode byte, longjmp on error.
687	*
688	* @returns The opcode byte.
689	* @param pVCpu The cross context virtual CPU structure of the calling thread.
690	*/
691	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
692	{
693	# ifdef IEM_WITH_CODE_TLB
694	uint8_t bRet;
695	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
696	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
697	if (RT_LIKELY( pbBuf != NULL
698	&& offBuf < pVCpu->iem.s.cbInstrBuf))
699	{
700	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
701	bRet = pbBuf[offBuf];
702	}
703	else
704	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
705	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
706	Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
707	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
708	# endif
709	return bRet;
710
711	# else /* !IEM_WITH_CODE_TLB */
712	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
713	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
714	{
715	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
716	return pVCpu->iem.s.abOpcode[offOpcode];
717	}
718	return iemOpcodeGetNextU8SlowJmp(pVCpu);
719	# endif
720	}
721
722	# endif /* IEM_WITH_SETJMP */
723
724	/**
725	* Fetches the next opcode byte, returns automatically on failure.
726	*
727	* @param a_pu8 Where to return the opcode byte.
728	* @remark Implicitly references pVCpu.
729	*/
730	# ifndef IEM_WITH_SETJMP
731	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
732	do \
733	{ \
734	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
735	if (rcStrict2 == VINF_SUCCESS) \
736	{ /* likely */ } \
737	else \
738	return rcStrict2; \
739	} while (0)
740	# else
741	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
742	# endif /* IEM_WITH_SETJMP */
743
744
745	# ifndef IEM_WITH_SETJMP
746	/**
747	* Fetches the next signed byte from the opcode stream.
748	*
749	* @returns Strict VBox status code.
750	* @param pVCpu The cross context virtual CPU structure of the calling thread.
751	* @param pi8 Where to return the signed byte.
752	*/
753	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
754	{
755	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
756	}
757	# endif /* !IEM_WITH_SETJMP */
758
759
760	/**
761	* Fetches the next signed byte from the opcode stream, returning automatically
762	* on failure.
763	*
764	* @param a_pi8 Where to return the signed byte.
765	* @remark Implicitly references pVCpu.
766	*/
767	# ifndef IEM_WITH_SETJMP
768	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
769	do \
770	{ \
771	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
772	if (rcStrict2 != VINF_SUCCESS) \
773	return rcStrict2; \
774	} while (0)
775	# else /* IEM_WITH_SETJMP */
776	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
777
778	# endif /* IEM_WITH_SETJMP */
779
780
781	# ifndef IEM_WITH_SETJMP
782	/**
783	* Fetches the next signed byte from the opcode stream, extending it to
784	* unsigned 16-bit.
785	*
786	* @returns Strict VBox status code.
787	* @param pVCpu The cross context virtual CPU structure of the calling thread.
788	* @param pu16 Where to return the unsigned word.
789	*/
790	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
791	{
792	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
793	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
794	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
795
796	*pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
797	pVCpu->iem.s.offOpcode = offOpcode + 1;
798	return VINF_SUCCESS;
799	}
800	# endif /* !IEM_WITH_SETJMP */
801
802	/**
803	* Fetches the next signed byte from the opcode stream and sign-extending it to
804	* a word, returning automatically on failure.
805	*
806	* @param a_pu16 Where to return the word.
807	* @remark Implicitly references pVCpu.
808	*/
809	# ifndef IEM_WITH_SETJMP
810	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
811	do \
812	{ \
813	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
814	if (rcStrict2 != VINF_SUCCESS) \
815	return rcStrict2; \
816	} while (0)
817	# else
818	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
819	# endif
820
821	# ifndef IEM_WITH_SETJMP
822	/**
823	* Fetches the next signed byte from the opcode stream, extending it to
824	* unsigned 32-bit.
825	*
826	* @returns Strict VBox status code.
827	* @param pVCpu The cross context virtual CPU structure of the calling thread.
828	* @param pu32 Where to return the unsigned dword.
829	*/
830	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
831	{
832	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
833	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
834	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
835
836	*pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
837	pVCpu->iem.s.offOpcode = offOpcode + 1;
838	return VINF_SUCCESS;
839	}
840	# endif /* !IEM_WITH_SETJMP */
841
842	/**
843	* Fetches the next signed byte from the opcode stream and sign-extending it to
844	* a word, returning automatically on failure.
845	*
846	* @param a_pu32 Where to return the word.
847	* @remark Implicitly references pVCpu.
848	*/
849	# ifndef IEM_WITH_SETJMP
850	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
851	do \
852	{ \
853	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
854	if (rcStrict2 != VINF_SUCCESS) \
855	return rcStrict2; \
856	} while (0)
857	# else
858	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
859	# endif
860
861
862	# ifndef IEM_WITH_SETJMP
863	/**
864	* Fetches the next signed byte from the opcode stream, extending it to
865	* unsigned 64-bit.
866	*
867	* @returns Strict VBox status code.
868	* @param pVCpu The cross context virtual CPU structure of the calling thread.
869	* @param pu64 Where to return the unsigned qword.
870	*/
871	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
872	{
873	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
874	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
875	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
876
877	*pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
878	pVCpu->iem.s.offOpcode = offOpcode + 1;
879	return VINF_SUCCESS;
880	}
881	# endif /* !IEM_WITH_SETJMP */
882
883	/**
884	* Fetches the next signed byte from the opcode stream and sign-extending it to
885	* a word, returning automatically on failure.
886	*
887	* @param a_pu64 Where to return the word.
888	* @remark Implicitly references pVCpu.
889	*/
890	# ifndef IEM_WITH_SETJMP
891	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
892	do \
893	{ \
894	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
895	if (rcStrict2 != VINF_SUCCESS) \
896	return rcStrict2; \
897	} while (0)
898	# else
899	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
900	# endif
901
902
903	# ifndef IEM_WITH_SETJMP
904
905	/**
906	* Fetches the next opcode word.
907	*
908	* @returns Strict VBox status code.
909	* @param pVCpu The cross context virtual CPU structure of the calling thread.
910	* @param pu16 Where to return the opcode word.
911	*/
912	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
913	{
914	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
915	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
916	{
917	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
918	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
919	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
920	# else
921	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
922	# endif
923	return VINF_SUCCESS;
924	}
925	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
926	}
927
928	# else /* IEM_WITH_SETJMP */
929
930	/**
931	* Fetches the next opcode word, longjmp on error.
932	*
933	* @returns The opcode word.
934	* @param pVCpu The cross context virtual CPU structure of the calling thread.
935	*/
936	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
937	{
938	# ifdef IEM_WITH_CODE_TLB
939	uint16_t u16Ret;
940	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
941	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
942	if (RT_LIKELY( pbBuf != NULL
943	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
944	{
945	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
946	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
947	u16Ret = (uint16_t const )&pbBuf[offBuf];
948	# else
949	u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
950	# endif
951	}
952	else
953	u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
954
955	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
956	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
957	Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
958	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
959	(uint16_t )&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
960	# else
961	pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
962	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
963	# endif
964	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
965	# endif
966
967	return u16Ret;
968
969	# else /* !IEM_WITH_CODE_TLB */
970	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
971	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
972	{
973	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
974	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
975	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
976	# else
977	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
978	# endif
979	}
980	return iemOpcodeGetNextU16SlowJmp(pVCpu);
981	# endif /* !IEM_WITH_CODE_TLB */
982	}
983
984	# endif /* IEM_WITH_SETJMP */
985
986	/**
987	* Fetches the next opcode word, returns automatically on failure.
988	*
989	* @param a_pu16 Where to return the opcode word.
990	* @remark Implicitly references pVCpu.
991	*/
992	# ifndef IEM_WITH_SETJMP
993	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
994	do \
995	{ \
996	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
997	if (rcStrict2 != VINF_SUCCESS) \
998	return rcStrict2; \
999	} while (0)
1000	# else
1001	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
1002	# endif
1003
1004	# ifndef IEM_WITH_SETJMP
1005	/**
1006	* Fetches the next opcode word, zero extending it to a double word.
1007	*
1008	* @returns Strict VBox status code.
1009	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1010	* @param pu32 Where to return the opcode double word.
1011	*/
1012	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1013	{
1014	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1015	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1016	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1017
1018	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1019	pVCpu->iem.s.offOpcode = offOpcode + 2;
1020	return VINF_SUCCESS;
1021	}
1022	# endif /* !IEM_WITH_SETJMP */
1023
1024	/**
1025	* Fetches the next opcode word and zero extends it to a double word, returns
1026	* automatically on failure.
1027	*
1028	* @param a_pu32 Where to return the opcode double word.
1029	* @remark Implicitly references pVCpu.
1030	*/
1031	# ifndef IEM_WITH_SETJMP
1032	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1033	do \
1034	{ \
1035	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1036	if (rcStrict2 != VINF_SUCCESS) \
1037	return rcStrict2; \
1038	} while (0)
1039	# else
1040	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1041	# endif
1042
1043	# ifndef IEM_WITH_SETJMP
1044	/**
1045	* Fetches the next opcode word, zero extending it to a quad word.
1046	*
1047	* @returns Strict VBox status code.
1048	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1049	* @param pu64 Where to return the opcode quad word.
1050	*/
1051	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1052	{
1053	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1054	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1055	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1056
1057	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1058	pVCpu->iem.s.offOpcode = offOpcode + 2;
1059	return VINF_SUCCESS;
1060	}
1061	# endif /* !IEM_WITH_SETJMP */
1062
1063	/**
1064	* Fetches the next opcode word and zero extends it to a quad word, returns
1065	* automatically on failure.
1066	*
1067	* @param a_pu64 Where to return the opcode quad word.
1068	* @remark Implicitly references pVCpu.
1069	*/
1070	# ifndef IEM_WITH_SETJMP
1071	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1072	do \
1073	{ \
1074	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1075	if (rcStrict2 != VINF_SUCCESS) \
1076	return rcStrict2; \
1077	} while (0)
1078	# else
1079	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1080	# endif
1081
1082
1083	# ifndef IEM_WITH_SETJMP
1084	/**
1085	* Fetches the next signed word from the opcode stream.
1086	*
1087	* @returns Strict VBox status code.
1088	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1089	* @param pi16 Where to return the signed word.
1090	*/
1091	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1092	{
1093	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1094	}
1095	# endif /* !IEM_WITH_SETJMP */
1096
1097
1098	/**
1099	* Fetches the next signed word from the opcode stream, returning automatically
1100	* on failure.
1101	*
1102	* @param a_pi16 Where to return the signed word.
1103	* @remark Implicitly references pVCpu.
1104	*/
1105	# ifndef IEM_WITH_SETJMP
1106	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1107	do \
1108	{ \
1109	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1110	if (rcStrict2 != VINF_SUCCESS) \
1111	return rcStrict2; \
1112	} while (0)
1113	# else
1114	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1115	# endif
1116
1117	# ifndef IEM_WITH_SETJMP
1118
1119	/**
1120	* Fetches the next opcode dword.
1121	*
1122	* @returns Strict VBox status code.
1123	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1124	* @param pu32 Where to return the opcode double word.
1125	*/
1126	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1127	{
1128	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1129	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1130	{
1131	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1132	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1133	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1134	# else
1135	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1136	pVCpu->iem.s.abOpcode[offOpcode + 1],
1137	pVCpu->iem.s.abOpcode[offOpcode + 2],
1138	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1139	# endif
1140	return VINF_SUCCESS;
1141	}
1142	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1143	}
1144
1145	# else /* IEM_WITH_SETJMP */
1146
1147	/**
1148	* Fetches the next opcode dword, longjmp on error.
1149	*
1150	* @returns The opcode dword.
1151	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1152	*/
1153	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1154	{
1155	# ifdef IEM_WITH_CODE_TLB
1156	uint32_t u32Ret;
1157	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1158	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1159	if (RT_LIKELY( pbBuf != NULL
1160	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1161	{
1162	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1163	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1164	u32Ret = (uint32_t const )&pbBuf[offBuf];
1165	# else
1166	u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1167	pbBuf[offBuf + 1],
1168	pbBuf[offBuf + 2],
1169	pbBuf[offBuf + 3]);
1170	# endif
1171	}
1172	else
1173	u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1174
1175	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1176	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1177	Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1178	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1179	(uint32_t )&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1180	# else
1181	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1182	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1183	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1184	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1185	# endif
1186	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1187	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1188
1189	return u32Ret;
1190
1191	# else /* !IEM_WITH_CODE_TLB */
1192	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1193	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1194	{
1195	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1196	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1197	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1198	# else
1199	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1200	pVCpu->iem.s.abOpcode[offOpcode + 1],
1201	pVCpu->iem.s.abOpcode[offOpcode + 2],
1202	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1203	# endif
1204	}
1205	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1206	# endif
1207	}
1208
1209	# endif /* IEM_WITH_SETJMP */
1210
1211	/**
1212	* Fetches the next opcode dword, returns automatically on failure.
1213	*
1214	* @param a_pu32 Where to return the opcode dword.
1215	* @remark Implicitly references pVCpu.
1216	*/
1217	# ifndef IEM_WITH_SETJMP
1218	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1219	do \
1220	{ \
1221	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1222	if (rcStrict2 != VINF_SUCCESS) \
1223	return rcStrict2; \
1224	} while (0)
1225	# else
1226	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1227	# endif
1228
1229	# ifndef IEM_WITH_SETJMP
1230	/**
1231	* Fetches the next opcode dword, zero extending it to a quad word.
1232	*
1233	* @returns Strict VBox status code.
1234	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1235	* @param pu64 Where to return the opcode quad word.
1236	*/
1237	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1238	{
1239	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1240	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1241	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1242
1243	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1244	pVCpu->iem.s.abOpcode[offOpcode + 1],
1245	pVCpu->iem.s.abOpcode[offOpcode + 2],
1246	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1247	pVCpu->iem.s.offOpcode = offOpcode + 4;
1248	return VINF_SUCCESS;
1249	}
1250	# endif /* !IEM_WITH_SETJMP */
1251
1252	/**
1253	* Fetches the next opcode dword and zero extends it to a quad word, returns
1254	* automatically on failure.
1255	*
1256	* @param a_pu64 Where to return the opcode quad word.
1257	* @remark Implicitly references pVCpu.
1258	*/
1259	# ifndef IEM_WITH_SETJMP
1260	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1261	do \
1262	{ \
1263	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1264	if (rcStrict2 != VINF_SUCCESS) \
1265	return rcStrict2; \
1266	} while (0)
1267	# else
1268	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1269	# endif
1270
1271
1272	# ifndef IEM_WITH_SETJMP
1273	/**
1274	* Fetches the next signed double word from the opcode stream.
1275	*
1276	* @returns Strict VBox status code.
1277	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1278	* @param pi32 Where to return the signed double word.
1279	*/
1280	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1281	{
1282	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1283	}
1284	# endif
1285
1286	/**
1287	* Fetches the next signed double word from the opcode stream, returning
1288	* automatically on failure.
1289	*
1290	* @param a_pi32 Where to return the signed double word.
1291	* @remark Implicitly references pVCpu.
1292	*/
1293	# ifndef IEM_WITH_SETJMP
1294	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1295	do \
1296	{ \
1297	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1298	if (rcStrict2 != VINF_SUCCESS) \
1299	return rcStrict2; \
1300	} while (0)
1301	# else
1302	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1303	# endif
1304
1305	# ifndef IEM_WITH_SETJMP
1306	/**
1307	* Fetches the next opcode dword, sign extending it into a quad word.
1308	*
1309	* @returns Strict VBox status code.
1310	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1311	* @param pu64 Where to return the opcode quad word.
1312	*/
1313	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1314	{
1315	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1316	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1317	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1318
1319	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1320	pVCpu->iem.s.abOpcode[offOpcode + 1],
1321	pVCpu->iem.s.abOpcode[offOpcode + 2],
1322	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1323	*pu64 = (uint64_t)(int64_t)i32;
1324	pVCpu->iem.s.offOpcode = offOpcode + 4;
1325	return VINF_SUCCESS;
1326	}
1327	# endif /* !IEM_WITH_SETJMP */
1328
1329	/**
1330	* Fetches the next opcode double word and sign extends it to a quad word,
1331	* returns automatically on failure.
1332	*
1333	* @param a_pu64 Where to return the opcode quad word.
1334	* @remark Implicitly references pVCpu.
1335	*/
1336	# ifndef IEM_WITH_SETJMP
1337	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1338	do \
1339	{ \
1340	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1341	if (rcStrict2 != VINF_SUCCESS) \
1342	return rcStrict2; \
1343	} while (0)
1344	# else
1345	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1346	# endif
1347
1348	# ifndef IEM_WITH_SETJMP
1349
1350	/**
1351	* Fetches the next opcode qword.
1352	*
1353	* @returns Strict VBox status code.
1354	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1355	* @param pu64 Where to return the opcode qword.
1356	*/
1357	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1358	{
1359	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1360	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1361	{
1362	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1363	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1364	# else
1365	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1366	pVCpu->iem.s.abOpcode[offOpcode + 1],
1367	pVCpu->iem.s.abOpcode[offOpcode + 2],
1368	pVCpu->iem.s.abOpcode[offOpcode + 3],
1369	pVCpu->iem.s.abOpcode[offOpcode + 4],
1370	pVCpu->iem.s.abOpcode[offOpcode + 5],
1371	pVCpu->iem.s.abOpcode[offOpcode + 6],
1372	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1373	# endif
1374	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1375	return VINF_SUCCESS;
1376	}
1377	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1378	}
1379
1380	# else /* IEM_WITH_SETJMP */
1381
1382	/**
1383	* Fetches the next opcode qword, longjmp on error.
1384	*
1385	* @returns The opcode qword.
1386	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1387	*/
1388	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1389	{
1390	# ifdef IEM_WITH_CODE_TLB
1391	uint64_t u64Ret;
1392	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1393	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1394	if (RT_LIKELY( pbBuf != NULL
1395	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1396	{
1397	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1398	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1399	u64Ret = (uint64_t const )&pbBuf[offBuf];
1400	# else
1401	u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1402	pbBuf[offBuf + 1],
1403	pbBuf[offBuf + 2],
1404	pbBuf[offBuf + 3],
1405	pbBuf[offBuf + 4],
1406	pbBuf[offBuf + 5],
1407	pbBuf[offBuf + 6],
1408	pbBuf[offBuf + 7]);
1409	# endif
1410	}
1411	else
1412	u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1413
1414	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1415	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1416	Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1417	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1418	(uint64_t )&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1419	# else
1420	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1421	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1422	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1423	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1424	pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1425	pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1426	pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1427	pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1428	# endif
1429	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1430	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1431
1432	return u64Ret;
1433
1434	# else /* !IEM_WITH_CODE_TLB */
1435	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1436	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1437	{
1438	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1439	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1440	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1441	# else
1442	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1443	pVCpu->iem.s.abOpcode[offOpcode + 1],
1444	pVCpu->iem.s.abOpcode[offOpcode + 2],
1445	pVCpu->iem.s.abOpcode[offOpcode + 3],
1446	pVCpu->iem.s.abOpcode[offOpcode + 4],
1447	pVCpu->iem.s.abOpcode[offOpcode + 5],
1448	pVCpu->iem.s.abOpcode[offOpcode + 6],
1449	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1450	# endif
1451	}
1452	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1453	# endif /* !IEM_WITH_CODE_TLB */
1454	}
1455
1456	# endif /* IEM_WITH_SETJMP */
1457
1458	/**
1459	* Fetches the next opcode quad word, returns automatically on failure.
1460	*
1461	* @param a_pu64 Where to return the opcode quad word.
1462	* @remark Implicitly references pVCpu.
1463	*/
1464	# ifndef IEM_WITH_SETJMP
1465	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1466	do \
1467	{ \
1468	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1469	if (rcStrict2 != VINF_SUCCESS) \
1470	return rcStrict2; \
1471	} while (0)
1472	# else
1473	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1474	# endif
1475
1476	/**
1477	* For fetching the opcode bytes for an ModR/M effective address, but throw
1478	* away the result.
1479	*
1480	* This is used when decoding undefined opcodes and such where we want to avoid
1481	* unnecessary MC blocks.
1482	*
1483	* @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1484	* used instead. At least for now...
1485	*/
1486	# ifndef IEM_WITH_SETJMP
1487	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1488	RTGCPTR GCPtrEff; \
1489	VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1490	if (rcStrict != VINF_SUCCESS) \
1491	return rcStrict; \
1492	} while (0)
1493	# else
1494	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1495	(void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1496	} while (0)
1497	# endif
1498
1499	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1500
1501
1502	/** @name Misc Worker Functions.
1503	* @{
1504	*/
1505
1506	/**
1507	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1508	* not (kind of obsolete now).
1509	*
1510	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1511	*/
1512	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1513
1514	/**
1515	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1516	*
1517	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1518	* @param a_fEfl The new EFLAGS.
1519	*/
1520	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1521
1522
1523	/**
1524	* Loads a NULL data selector into a selector register, both the hidden and
1525	* visible parts, in protected mode.
1526	*
1527	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1528	* @param pSReg Pointer to the segment register.
1529	* @param uRpl The RPL.
1530	*/
1531	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1532	{
1533	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1534	* data selector in protected mode. */
1535	pSReg->Sel = uRpl;
1536	pSReg->ValidSel = uRpl;
1537	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1538	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1539	{
1540	/* VT-x (Intel 3960x) observed doing something like this. */
1541	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1542	pSReg->u32Limit = UINT32_MAX;
1543	pSReg->u64Base = 0;
1544	}
1545	else
1546	{
1547	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1548	pSReg->u32Limit = 0;
1549	pSReg->u64Base = 0;
1550	}
1551	}
1552
1553	/** @} */
1554
1555
1556	/*
1557	*
1558	* Helpers routines.
1559	* Helpers routines.
1560	* Helpers routines.
1561	*
1562	*/
1563
1564	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1565
1566	/**
1567	* Recalculates the effective operand size.
1568	*
1569	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1570	*/
1571	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1572	{
1573	switch (IEM_GET_CPU_MODE(pVCpu))
1574	{
1575	case IEMMODE_16BIT:
1576	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1577	break;
1578	case IEMMODE_32BIT:
1579	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1580	break;
1581	case IEMMODE_64BIT:
1582	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1583	{
1584	case 0:
1585	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1586	break;
1587	case IEM_OP_PRF_SIZE_OP:
1588	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1589	break;
1590	case IEM_OP_PRF_SIZE_REX_W:
1591	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1592	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1593	break;
1594	}
1595	break;
1596	default:
1597	AssertFailed();
1598	}
1599	}
1600
1601
1602	/**
1603	* Sets the default operand size to 64-bit and recalculates the effective
1604	* operand size.
1605	*
1606	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1607	*/
1608	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1609	{
1610	Assert(IEM_IS_64BIT_CODE(pVCpu));
1611	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1612	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1613	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1614	else
1615	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1616	}
1617
1618
1619	/**
1620	* Sets the default operand size to 64-bit and recalculates the effective
1621	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1622	*
1623	* This is for the relative jumps.
1624	*
1625	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1626	*/
1627	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1628	{
1629	Assert(IEM_IS_64BIT_CODE(pVCpu));
1630	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1631	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1632	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1633	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1634	else
1635	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1636	}
1637
1638	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1639
1640
1641
1642	/** @name Register Access.
1643	* @{
1644	*/
1645
1646	/**
1647	* Gets a reference (pointer) to the specified hidden segment register.
1648	*
1649	* @returns Hidden register reference.
1650	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1651	* @param iSegReg The segment register.
1652	*/
1653	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1654	{
1655	Assert(iSegReg < X86_SREG_COUNT);
1656	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1657	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1658
1659	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1660	return pSReg;
1661	}
1662
1663
1664	/**
1665	* Ensures that the given hidden segment register is up to date.
1666	*
1667	* @returns Hidden register reference.
1668	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1669	* @param pSReg The segment register.
1670	*/
1671	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1672	{
1673	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1674	NOREF(pVCpu);
1675	return pSReg;
1676	}
1677
1678
1679	/**
1680	* Gets a reference (pointer) to the specified segment register (the selector
1681	* value).
1682	*
1683	* @returns Pointer to the selector variable.
1684	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1685	* @param iSegReg The segment register.
1686	*/
1687	DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1688	{
1689	Assert(iSegReg < X86_SREG_COUNT);
1690	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1691	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1692	}
1693
1694
1695	/**
1696	* Fetches the selector value of a segment register.
1697	*
1698	* @returns The selector value.
1699	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1700	* @param iSegReg The segment register.
1701	*/
1702	DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1703	{
1704	Assert(iSegReg < X86_SREG_COUNT);
1705	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1706	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1707	}
1708
1709
1710	/**
1711	* Fetches the base address value of a segment register.
1712	*
1713	* @returns The selector value.
1714	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1715	* @param iSegReg The segment register.
1716	*/
1717	DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1718	{
1719	Assert(iSegReg < X86_SREG_COUNT);
1720	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1721	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1722	}
1723
1724
1725	/**
1726	* Gets a reference (pointer) to the specified general purpose register.
1727	*
1728	* @returns Register reference.
1729	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1730	* @param iReg The general purpose register.
1731	*/
1732	DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1733	{
1734	Assert(iReg < 16);
1735	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1736	}
1737
1738
1739	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1740	/**
1741	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1742	*
1743	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1744	*
1745	* @returns Register reference.
1746	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1747	* @param iReg The register.
1748	*/
1749	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1750	{
1751	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1752	{
1753	Assert(iReg < 16);
1754	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1755	}
1756	/* high 8-bit register. */
1757	Assert(iReg < 8);
1758	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1759	}
1760	#endif
1761
1762
1763	/**
1764	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1765	* alternative version with extended (20) register index.
1766	*
1767	* @returns Register reference.
1768	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1769	* @param iRegEx The register. The 16 first are regular ones,
1770	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1771	*/
1772	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1773	{
1774	/** @todo This could be done by double indexing on little endian hosts:
1775	* return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1776	if (iRegEx < 16)
1777	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1778
1779	/* high 8-bit register. */
1780	Assert(iRegEx < 20);
1781	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1782	}
1783
1784
1785	/**
1786	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1787	*
1788	* @returns Register reference.
1789	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1790	* @param iReg The register.
1791	*/
1792	DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1793	{
1794	Assert(iReg < 16);
1795	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1796	}
1797
1798
1799	/**
1800	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1801	*
1802	* @returns Register reference.
1803	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1804	* @param iReg The register.
1805	*/
1806	DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1807	{
1808	Assert(iReg < 16);
1809	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1810	}
1811
1812
1813	/**
1814	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1815	*
1816	* @returns Register reference.
1817	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1818	* @param iReg The register.
1819	*/
1820	DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1821	{
1822	Assert(iReg < 16);
1823	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1824	}
1825
1826
1827	/**
1828	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1829	*
1830	* @returns Register reference.
1831	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1832	* @param iReg The register.
1833	*/
1834	DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1835	{
1836	Assert(iReg < 64);
1837	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1838	}
1839
1840
1841	/**
1842	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1843	*
1844	* @returns Register reference.
1845	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1846	* @param iReg The register.
1847	*/
1848	DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1849	{
1850	Assert(iReg < 16);
1851	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1852	}
1853
1854
1855	/**
1856	* Gets a reference (pointer) to the specified segment register's base address.
1857	*
1858	* @returns Segment register base address reference.
1859	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1860	* @param iSegReg The segment selector.
1861	*/
1862	DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1863	{
1864	Assert(iSegReg < X86_SREG_COUNT);
1865	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1866	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1867	}
1868
1869
1870	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1871	/**
1872	* Fetches the value of a 8-bit general purpose register.
1873	*
1874	* @returns The register value.
1875	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1876	* @param iReg The register.
1877	*/
1878	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1879	{
1880	return *iemGRegRefU8(pVCpu, iReg);
1881	}
1882	#endif
1883
1884
1885	/**
1886	* Fetches the value of a 8-bit general purpose register, alternative version
1887	* with extended (20) register index.
1888
1889	* @returns The register value.
1890	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1891	* @param iRegEx The register. The 16 first are regular ones,
1892	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1893	*/
1894	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1895	{
1896	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1897	}
1898
1899
1900	/**
1901	* Fetches the value of a 16-bit general purpose register.
1902	*
1903	* @returns The register value.
1904	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1905	* @param iReg The register.
1906	*/
1907	DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1908	{
1909	Assert(iReg < 16);
1910	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1911	}
1912
1913
1914	/**
1915	* Fetches the value of a 32-bit general purpose register.
1916	*
1917	* @returns The register value.
1918	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1919	* @param iReg The register.
1920	*/
1921	DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1922	{
1923	Assert(iReg < 16);
1924	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1925	}
1926
1927
1928	/**
1929	* Fetches the value of a 64-bit general purpose register.
1930	*
1931	* @returns The register value.
1932	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1933	* @param iReg The register.
1934	*/
1935	DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1936	{
1937	Assert(iReg < 16);
1938	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1939	}
1940
1941
1942	/**
1943	* Stores a 16-bit value to a general purpose register.
1944	*
1945	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1946	* @param iReg The register.
1947	* @param uValue The value to store.
1948	*/
1949	DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1950	{
1951	Assert(iReg < 16);
1952	pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1953	}
1954
1955
1956	/**
1957	* Stores a 32-bit value to a general purpose register, implicitly clearing high
1958	* values.
1959	*
1960	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1961	* @param iReg The register.
1962	* @param uValue The value to store.
1963	*/
1964	DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1965	{
1966	Assert(iReg < 16);
1967	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1968	}
1969
1970
1971	/**
1972	* Stores a 64-bit value to a general purpose register.
1973	*
1974	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1975	* @param iReg The register.
1976	* @param uValue The value to store.
1977	*/
1978	DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1979	{
1980	Assert(iReg < 16);
1981	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1982	}
1983
1984
1985	/**
1986	* Get the address of the top of the stack.
1987	*
1988	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1989	*/
1990	DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1991	{
1992	if (IEM_IS_64BIT_CODE(pVCpu))
1993	return pVCpu->cpum.GstCtx.rsp;
1994	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1995	return pVCpu->cpum.GstCtx.esp;
1996	return pVCpu->cpum.GstCtx.sp;
1997	}
1998
1999
2000	/**
2001	* Updates the RIP/EIP/IP to point to the next instruction.
2002	*
2003	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2004	* @param cbInstr The number of bytes to add.
2005	*/
2006	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2007	{
2008	/*
2009	* Advance RIP.
2010	*
2011	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
2012	* while in all other modes except LM64 the updates are 32-bit. This means
2013	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
2014	* 4GB and 64KB rollovers, and decide whether anything needs masking.
2015	*
2016	* See PC wrap around tests in bs3-cpu-weird-1.
2017	*/
2018	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2019	uint64_t const uRipNext = uRipPrev + cbInstr;
2020	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
2021	\|\| IEM_IS_64BIT_CODE(pVCpu)))
2022	pVCpu->cpum.GstCtx.rip = uRipNext;
2023	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2024	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2025	else
2026	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2027	}
2028
2029
2030	/**
2031	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2032	* following EFLAGS bits are set:
2033	* - X86_EFL_RF - clear it.
2034	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2035	* - X86_EFL_TF - generate single step \#DB trap.
2036	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2037	* instruction).
2038	*
2039	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2040	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2041	* takes priority over both NMIs and hardware interrupts. So, neither is
2042	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2043	* either unsupported will be triggered on-top of any \#DB raised here.)
2044	*
2045	* The RF flag only needs to be cleared here as it only suppresses instruction
2046	* breakpoints which are not raised here (happens synchronously during
2047	* instruction fetching).
2048	*
2049	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2050	* status has no bearing on whether \#DB exceptions are raised.
2051	*
2052	* @note This must NOT be called by the two instructions setting the
2053	* CPUMCTX_INHIBIT_SHADOW_SS flag.
2054	*
2055	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2056	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2057	* Stacks}
2058	*/
2059	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2060	{
2061	/*
2062	* Normally we're just here to clear RF and/or interrupt shadow bits.
2063	*/
2064	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
2065	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
2066	else
2067	{
2068	/*
2069	* Raise a #DB or/and DBGF event.
2070	*/
2071	VBOXSTRICTRC rcStrict;
2072	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
2073	{
2074	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2075	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2076	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2077	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
2078	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2079	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2080	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2081	pVCpu->cpum.GstCtx.rflags.uBoth));
2082
2083	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
2084	rcStrict = iemRaiseDebugException(pVCpu);
2085
2086	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2087	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2088	{
2089	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2090	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2091	}
2092	}
2093	else
2094	{
2095	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2096	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2097	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2098	}
2099	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2100	return rcStrict;
2101	}
2102	return VINF_SUCCESS;
2103	}
2104
2105
2106	/**
2107	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2108	*
2109	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2110	*/
2111	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2112	{
2113	/*
2114	* We assume that most of the time nothing actually needs doing here.
2115	*/
2116	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2117	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2118	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
2119	return VINF_SUCCESS;
2120	return iemFinishInstructionWithFlagsSet(pVCpu);
2121	}
2122
2123
2124	/**
2125	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2126	* and CPUMCTX_INHIBIT_SHADOW.
2127	*
2128	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2129	* @param cbInstr The number of bytes to add.
2130	*/
2131	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2132	{
2133	iemRegAddToRip(pVCpu, cbInstr);
2134	return iemRegFinishClearingRF(pVCpu);
2135	}
2136
2137
2138	/**
2139	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
2140	* and CPUMCTX_INHIBIT_SHADOW.
2141	*
2142	* Only called from 64-bit code.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2145	* @param cbInstr The number of bytes to add.
2146	*/
2147	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2148	{
2149	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2150	return iemRegFinishClearingRF(pVCpu);
2151	}
2152
2153
2154	/**
2155	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2156	* CPUMCTX_INHIBIT_SHADOW.
2157	*
2158	* This is never from 64-bit code.
2159	*
2160	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2161	* @param cbInstr The number of bytes to add.
2162	*/
2163	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2164	{
2165	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2166	return iemRegFinishClearingRF(pVCpu);
2167	}
2168
2169
2170	/**
2171	* Updates the IP to point to the next instruction and clears EFLAGS.RF and
2172	* CPUMCTX_INHIBIT_SHADOW.
2173	*
2174	* This is only ever used from 16-bit code on a pre-386 CPU.
2175	*
2176	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2177	* @param cbInstr The number of bytes to add.
2178	*/
2179	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2180	{
2181	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2182	return iemRegFinishClearingRF(pVCpu);
2183	}
2184
2185
2186	/**
2187	* Adds a 8-bit signed jump offset to RIP from 64-bit code.
2188	*
2189	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2190	* segment limit.
2191	*
2192	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2193	* @param cbInstr Instruction size.
2194	* @param offNextInstr The offset of the next instruction.
2195	* @param enmEffOpSize Effective operand size.
2196	*/
2197	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2198	IEMMODE enmEffOpSize) RT_NOEXCEPT
2199	{
2200	Assert(IEM_IS_64BIT_CODE(pVCpu));
2201	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2202
2203	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2204	if (enmEffOpSize == IEMMODE_16BIT)
2205	uNewRip &= UINT16_MAX;
2206
2207	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2208	pVCpu->cpum.GstCtx.rip = uNewRip;
2209	else
2210	return iemRaiseGeneralProtectionFault0(pVCpu);
2211
2212	#ifndef IEM_WITH_CODE_TLB
2213	iemOpcodeFlushLight(pVCpu, cbInstr);
2214	#endif
2215
2216	/*
2217	* Clear RF and finish the instruction (maybe raise #DB).
2218	*/
2219	return iemRegFinishClearingRF(pVCpu);
2220	}
2221
2222
2223	/**
2224	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2225	* code (never 64-bit).
2226	*
2227	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2228	* segment limit.
2229	*
2230	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2231	* @param cbInstr Instruction size.
2232	* @param offNextInstr The offset of the next instruction.
2233	* @param enmEffOpSize Effective operand size.
2234	*/
2235	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2236	IEMMODE enmEffOpSize) RT_NOEXCEPT
2237	{
2238	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2239	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2240
2241	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2242	if (enmEffOpSize == IEMMODE_16BIT)
2243	uNewEip &= UINT16_MAX;
2244	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2245	pVCpu->cpum.GstCtx.rip = uNewEip;
2246	else
2247	return iemRaiseGeneralProtectionFault0(pVCpu);
2248
2249	#ifndef IEM_WITH_CODE_TLB
2250	iemOpcodeFlushLight(pVCpu, cbInstr);
2251	#endif
2252
2253	/*
2254	* Clear RF and finish the instruction (maybe raise #DB).
2255	*/
2256	return iemRegFinishClearingRF(pVCpu);
2257	}
2258
2259
2260	/**
2261	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2262	*
2263	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2264	* segment limit.
2265	*
2266	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2267	* @param cbInstr Instruction size.
2268	* @param offNextInstr The offset of the next instruction.
2269	*/
2270	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2271	int8_t offNextInstr) RT_NOEXCEPT
2272	{
2273	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2274
2275	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2276	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2277	pVCpu->cpum.GstCtx.rip = uNewIp;
2278	else
2279	return iemRaiseGeneralProtectionFault0(pVCpu);
2280
2281	#ifndef IEM_WITH_CODE_TLB
2282	iemOpcodeFlushLight(pVCpu, cbInstr);
2283	#endif
2284
2285	/*
2286	* Clear RF and finish the instruction (maybe raise #DB).
2287	*/
2288	return iemRegFinishClearingRF(pVCpu);
2289	}
2290
2291
2292	/**
2293	* Adds a 16-bit signed jump offset to RIP from 64-bit code.
2294	*
2295	* @returns Strict VBox status code.
2296	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2297	* @param cbInstr Instruction size.
2298	* @param offNextInstr The offset of the next instruction.
2299	*/
2300	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2301	int16_t offNextInstr) RT_NOEXCEPT
2302	{
2303	Assert(IEM_IS_64BIT_CODE(pVCpu));
2304
2305	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2306
2307	#ifndef IEM_WITH_CODE_TLB
2308	iemOpcodeFlushLight(pVCpu, cbInstr);
2309	#endif
2310
2311	/*
2312	* Clear RF and finish the instruction (maybe raise #DB).
2313	*/
2314	return iemRegFinishClearingRF(pVCpu);
2315	}
2316
2317
2318	/**
2319	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2320	*
2321	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2322	* segment limit.
2323	*
2324	* @returns Strict VBox status code.
2325	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2326	* @param cbInstr Instruction size.
2327	* @param offNextInstr The offset of the next instruction.
2328	*
2329	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2330	* identical.
2331	*/
2332	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2333	int16_t offNextInstr) RT_NOEXCEPT
2334	{
2335	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2336
2337	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2338	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2339	pVCpu->cpum.GstCtx.rip = uNewIp;
2340	else
2341	return iemRaiseGeneralProtectionFault0(pVCpu);
2342
2343	#ifndef IEM_WITH_CODE_TLB
2344	iemOpcodeFlushLight(pVCpu, cbInstr);
2345	#endif
2346
2347	/*
2348	* Clear RF and finish the instruction (maybe raise #DB).
2349	*/
2350	return iemRegFinishClearingRF(pVCpu);
2351	}
2352
2353
2354	/**
2355	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2356	*
2357	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2358	* segment limit.
2359	*
2360	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2361	* only alternative for relative jumps in 64-bit code and that is already
2362	* handled in the decoder stage.
2363	*
2364	* @returns Strict VBox status code.
2365	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2366	* @param cbInstr Instruction size.
2367	* @param offNextInstr The offset of the next instruction.
2368	*/
2369	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2370	int32_t offNextInstr) RT_NOEXCEPT
2371	{
2372	Assert(IEM_IS_64BIT_CODE(pVCpu));
2373
2374	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2375	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2376	pVCpu->cpum.GstCtx.rip = uNewRip;
2377	else
2378	return iemRaiseGeneralProtectionFault0(pVCpu);
2379
2380	#ifndef IEM_WITH_CODE_TLB
2381	iemOpcodeFlushLight(pVCpu, cbInstr);
2382	#endif
2383
2384	/*
2385	* Clear RF and finish the instruction (maybe raise #DB).
2386	*/
2387	return iemRegFinishClearingRF(pVCpu);
2388	}
2389
2390
2391	/**
2392	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2393	*
2394	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2395	* segment limit.
2396	*
2397	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2398	* only alternative for relative jumps in 32-bit code and that is already
2399	* handled in the decoder stage.
2400	*
2401	* @returns Strict VBox status code.
2402	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2403	* @param cbInstr Instruction size.
2404	* @param offNextInstr The offset of the next instruction.
2405	*/
2406	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2407	int32_t offNextInstr) RT_NOEXCEPT
2408	{
2409	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2410	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2411
2412	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2413	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2414	pVCpu->cpum.GstCtx.rip = uNewEip;
2415	else
2416	return iemRaiseGeneralProtectionFault0(pVCpu);
2417
2418	#ifndef IEM_WITH_CODE_TLB
2419	iemOpcodeFlushLight(pVCpu, cbInstr);
2420	#endif
2421
2422	/*
2423	* Clear RF and finish the instruction (maybe raise #DB).
2424	*/
2425	return iemRegFinishClearingRF(pVCpu);
2426	}
2427
2428
2429	/**
2430	* Extended version of iemFinishInstructionWithFlagsSet that goes with
2431	* iemRegAddToRipAndFinishingClearingRfEx.
2432	*
2433	* See iemFinishInstructionWithFlagsSet() for details.
2434	*/
2435	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2436	{
2437	/*
2438	* Raise a #DB.
2439	*/
2440	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2441	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2442	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
2443	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2444	/** @todo Do we set all pending \#DB events, or just one? */
2445	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2446	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2447	pVCpu->cpum.GstCtx.rflags.uBoth));
2448	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
2449	return iemRaiseDebugException(pVCpu);
2450	}
2451
2452
2453	/**
2454	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2455	* others potentially updating EFLAGS.TF.
2456	*
2457	* The single step event must be generated using the TF value at the start of
2458	* the instruction, not the new value set by it.
2459	*
2460	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2461	* @param cbInstr The number of bytes to add.
2462	* @param fEflOld The EFLAGS at the start of the instruction
2463	* execution.
2464	*/
2465	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2466	{
2467	iemRegAddToRip(pVCpu, cbInstr);
2468	if (!(fEflOld & X86_EFL_TF))
2469	return iemRegFinishClearingRF(pVCpu);
2470	return iemFinishInstructionWithTfSet(pVCpu);
2471	}
2472
2473
2474	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2475	/**
2476	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2477	*
2478	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2479	*/
2480	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2481	{
2482	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2483	}
2484	#endif
2485
2486
2487	/**
2488	* Adds to the stack pointer.
2489	*
2490	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2491	* @param cbToAdd The number of bytes to add (8-bit!).
2492	*/
2493	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2494	{
2495	if (IEM_IS_64BIT_CODE(pVCpu))
2496	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2497	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2498	pVCpu->cpum.GstCtx.esp += cbToAdd;
2499	else
2500	pVCpu->cpum.GstCtx.sp += cbToAdd;
2501	}
2502
2503
2504	/**
2505	* Subtracts from the stack pointer.
2506	*
2507	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2508	* @param cbToSub The number of bytes to subtract (8-bit!).
2509	*/
2510	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2511	{
2512	if (IEM_IS_64BIT_CODE(pVCpu))
2513	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2514	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2515	pVCpu->cpum.GstCtx.esp -= cbToSub;
2516	else
2517	pVCpu->cpum.GstCtx.sp -= cbToSub;
2518	}
2519
2520
2521	/**
2522	* Adds to the temporary stack pointer.
2523	*
2524	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2525	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2526	* @param cbToAdd The number of bytes to add (16-bit).
2527	*/
2528	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2529	{
2530	if (IEM_IS_64BIT_CODE(pVCpu))
2531	pTmpRsp->u += cbToAdd;
2532	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2533	pTmpRsp->DWords.dw0 += cbToAdd;
2534	else
2535	pTmpRsp->Words.w0 += cbToAdd;
2536	}
2537
2538
2539	/**
2540	* Subtracts from the temporary stack pointer.
2541	*
2542	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2543	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2544	* @param cbToSub The number of bytes to subtract.
2545	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2546	* expecting that.
2547	*/
2548	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2549	{
2550	if (IEM_IS_64BIT_CODE(pVCpu))
2551	pTmpRsp->u -= cbToSub;
2552	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2553	pTmpRsp->DWords.dw0 -= cbToSub;
2554	else
2555	pTmpRsp->Words.w0 -= cbToSub;
2556	}
2557
2558
2559	/**
2560	* Calculates the effective stack address for a push of the specified size as
2561	* well as the new RSP value (upper bits may be masked).
2562	*
2563	* @returns Effective stack addressf for the push.
2564	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2565	* @param cbItem The size of the stack item to pop.
2566	* @param puNewRsp Where to return the new RSP value.
2567	*/
2568	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2569	{
2570	RTUINT64U uTmpRsp;
2571	RTGCPTR GCPtrTop;
2572	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2573
2574	if (IEM_IS_64BIT_CODE(pVCpu))
2575	GCPtrTop = uTmpRsp.u -= cbItem;
2576	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2577	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2578	else
2579	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2580	*puNewRsp = uTmpRsp.u;
2581	return GCPtrTop;
2582	}
2583
2584
2585	/**
2586	* Gets the current stack pointer and calculates the value after a pop of the
2587	* specified size.
2588	*
2589	* @returns Current stack pointer.
2590	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2591	* @param cbItem The size of the stack item to pop.
2592	* @param puNewRsp Where to return the new RSP value.
2593	*/
2594	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2595	{
2596	RTUINT64U uTmpRsp;
2597	RTGCPTR GCPtrTop;
2598	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2599
2600	if (IEM_IS_64BIT_CODE(pVCpu))
2601	{
2602	GCPtrTop = uTmpRsp.u;
2603	uTmpRsp.u += cbItem;
2604	}
2605	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2606	{
2607	GCPtrTop = uTmpRsp.DWords.dw0;
2608	uTmpRsp.DWords.dw0 += cbItem;
2609	}
2610	else
2611	{
2612	GCPtrTop = uTmpRsp.Words.w0;
2613	uTmpRsp.Words.w0 += cbItem;
2614	}
2615	*puNewRsp = uTmpRsp.u;
2616	return GCPtrTop;
2617	}
2618
2619
2620	/**
2621	* Calculates the effective stack address for a push of the specified size as
2622	* well as the new temporary RSP value (upper bits may be masked).
2623	*
2624	* @returns Effective stack addressf for the push.
2625	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2626	* @param pTmpRsp The temporary stack pointer. This is updated.
2627	* @param cbItem The size of the stack item to pop.
2628	*/
2629	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2630	{
2631	RTGCPTR GCPtrTop;
2632
2633	if (IEM_IS_64BIT_CODE(pVCpu))
2634	GCPtrTop = pTmpRsp->u -= cbItem;
2635	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2636	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2637	else
2638	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2639	return GCPtrTop;
2640	}
2641
2642
2643	/**
2644	* Gets the effective stack address for a pop of the specified size and
2645	* calculates and updates the temporary RSP.
2646	*
2647	* @returns Current stack pointer.
2648	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2649	* @param pTmpRsp The temporary stack pointer. This is updated.
2650	* @param cbItem The size of the stack item to pop.
2651	*/
2652	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2653	{
2654	RTGCPTR GCPtrTop;
2655	if (IEM_IS_64BIT_CODE(pVCpu))
2656	{
2657	GCPtrTop = pTmpRsp->u;
2658	pTmpRsp->u += cbItem;
2659	}
2660	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2661	{
2662	GCPtrTop = pTmpRsp->DWords.dw0;
2663	pTmpRsp->DWords.dw0 += cbItem;
2664	}
2665	else
2666	{
2667	GCPtrTop = pTmpRsp->Words.w0;
2668	pTmpRsp->Words.w0 += cbItem;
2669	}
2670	return GCPtrTop;
2671	}
2672
2673	/** @} */
2674
2675
2676	/** @name FPU access and helpers.
2677	*
2678	* @{
2679	*/
2680
2681
2682	/**
2683	* Hook for preparing to use the host FPU.
2684	*
2685	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2686	*
2687	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2688	*/
2689	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2690	{
2691	#ifdef IN_RING3
2692	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2693	#else
2694	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2695	#endif
2696	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2697	}
2698
2699
2700	/**
2701	* Hook for preparing to use the host FPU for SSE.
2702	*
2703	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2704	*
2705	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2706	*/
2707	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2708	{
2709	iemFpuPrepareUsage(pVCpu);
2710	}
2711
2712
2713	/**
2714	* Hook for preparing to use the host FPU for AVX.
2715	*
2716	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2717	*
2718	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2719	*/
2720	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2721	{
2722	iemFpuPrepareUsage(pVCpu);
2723	}
2724
2725
2726	/**
2727	* Hook for actualizing the guest FPU state before the interpreter reads it.
2728	*
2729	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2730	*
2731	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2732	*/
2733	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2734	{
2735	#ifdef IN_RING3
2736	NOREF(pVCpu);
2737	#else
2738	CPUMRZFpuStateActualizeForRead(pVCpu);
2739	#endif
2740	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2741	}
2742
2743
2744	/**
2745	* Hook for actualizing the guest FPU state before the interpreter changes it.
2746	*
2747	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2748	*
2749	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2750	*/
2751	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2752	{
2753	#ifdef IN_RING3
2754	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2755	#else
2756	CPUMRZFpuStateActualizeForChange(pVCpu);
2757	#endif
2758	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2759	}
2760
2761
2762	/**
2763	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2764	* only.
2765	*
2766	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2767	*
2768	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2769	*/
2770	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2771	{
2772	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2773	NOREF(pVCpu);
2774	#else
2775	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2776	#endif
2777	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2778	}
2779
2780
2781	/**
2782	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2783	* read+write.
2784	*
2785	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2786	*
2787	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2788	*/
2789	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2790	{
2791	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2792	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2793	#else
2794	CPUMRZFpuStateActualizeForChange(pVCpu);
2795	#endif
2796	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2797
2798	/* Make sure any changes are loaded the next time around. */
2799	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2800	}
2801
2802
2803	/**
2804	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2805	* only.
2806	*
2807	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2808	*
2809	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2810	*/
2811	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2812	{
2813	#ifdef IN_RING3
2814	NOREF(pVCpu);
2815	#else
2816	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2817	#endif
2818	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2819	}
2820
2821
2822	/**
2823	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2824	* read+write.
2825	*
2826	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2827	*
2828	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2829	*/
2830	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2831	{
2832	#ifdef IN_RING3
2833	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2834	#else
2835	CPUMRZFpuStateActualizeForChange(pVCpu);
2836	#endif
2837	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2838
2839	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2840	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2841	}
2842
2843
2844	/**
2845	* Stores a QNaN value into a FPU register.
2846	*
2847	* @param pReg Pointer to the register.
2848	*/
2849	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2850	{
2851	pReg->au32[0] = UINT32_C(0x00000000);
2852	pReg->au32[1] = UINT32_C(0xc0000000);
2853	pReg->au16[4] = UINT16_C(0xffff);
2854	}
2855
2856
2857	/**
2858	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
2859	*
2860	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2861	* @param pFpuCtx The FPU context.
2862	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2863	*/
2864	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2865	{
2866	Assert(uFpuOpcode != UINT16_MAX);
2867	pFpuCtx->FOP = uFpuOpcode;
2868	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2869	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2870	{
2871	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2872	* happens in real mode here based on the fnsave and fnstenv images. */
2873	pFpuCtx->CS = 0;
2874	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2875	}
2876	else if (!IEM_IS_LONG_MODE(pVCpu))
2877	{
2878	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2879	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2880	}
2881	else
2882	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2883	}
2884
2885
2886	/**
2887	* Marks the specified stack register as free (for FFREE).
2888	*
2889	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2890	* @param iStReg The register to free.
2891	*/
2892	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2893	{
2894	Assert(iStReg < 8);
2895	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2896	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2897	pFpuCtx->FTW &= ~RT_BIT(iReg);
2898	}
2899
2900
2901	/**
2902	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2903	*
2904	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2905	*/
2906	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2907	{
2908	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2909	uint16_t uFsw = pFpuCtx->FSW;
2910	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2911	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2912	uFsw &= ~X86_FSW_TOP_MASK;
2913	uFsw \|= uTop;
2914	pFpuCtx->FSW = uFsw;
2915	}
2916
2917
2918	/**
2919	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2920	*
2921	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2922	*/
2923	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2924	{
2925	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2926	uint16_t uFsw = pFpuCtx->FSW;
2927	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2928	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2929	uFsw &= ~X86_FSW_TOP_MASK;
2930	uFsw \|= uTop;
2931	pFpuCtx->FSW = uFsw;
2932	}
2933
2934
2935
2936
2937	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2938	{
2939	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2940	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2941	if (pFpuCtx->FTW & RT_BIT(iReg))
2942	return VINF_SUCCESS;
2943	return VERR_NOT_FOUND;
2944	}
2945
2946
2947	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2948	{
2949	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2950	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2951	if (pFpuCtx->FTW & RT_BIT(iReg))
2952	{
2953	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2954	return VINF_SUCCESS;
2955	}
2956	return VERR_NOT_FOUND;
2957	}
2958
2959
2960	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2961	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2962	{
2963	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2964	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2965	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2966	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2967	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2968	{
2969	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2970	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2971	return VINF_SUCCESS;
2972	}
2973	return VERR_NOT_FOUND;
2974	}
2975
2976
2977	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2978	{
2979	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2980	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2981	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2982	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2983	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2984	{
2985	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2986	return VINF_SUCCESS;
2987	}
2988	return VERR_NOT_FOUND;
2989	}
2990
2991
2992	/**
2993	* Rotates the stack registers when setting new TOS.
2994	*
2995	* @param pFpuCtx The FPU context.
2996	* @param iNewTop New TOS value.
2997	* @remarks We only do this to speed up fxsave/fxrstor which
2998	* arrange the FP registers in stack order.
2999	* MUST be done before writing the new TOS (FSW).
3000	*/
3001	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
3002	{
3003	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3004	RTFLOAT80U ar80Temp[8];
3005
3006	if (iOldTop == iNewTop)
3007	return;
3008
3009	/* Unscrew the stack and get it into 'native' order. */
3010	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
3011	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
3012	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
3013	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
3014	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
3015	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
3016	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
3017	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3018
3019	/* Now rotate the stack to the new position. */
3020	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3021	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3022	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3023	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3024	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3025	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3026	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3027	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3028	}
3029
3030
3031	/**
3032	* Updates the FPU exception status after FCW is changed.
3033	*
3034	* @param pFpuCtx The FPU context.
3035	*/
3036	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3037	{
3038	uint16_t u16Fsw = pFpuCtx->FSW;
3039	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3040	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
3041	else
3042	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
3043	pFpuCtx->FSW = u16Fsw;
3044	}
3045
3046
3047	/**
3048	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3049	*
3050	* @returns The full FTW.
3051	* @param pFpuCtx The FPU context.
3052	*/
3053	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3054	{
3055	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3056	uint16_t u16Ftw = 0;
3057	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3058	for (unsigned iSt = 0; iSt < 8; iSt++)
3059	{
3060	unsigned const iReg = (iSt + iTop) & 7;
3061	if (!(u8Ftw & RT_BIT(iReg)))
3062	u16Ftw \|= 3 << (iReg * 2); /* empty */
3063	else
3064	{
3065	uint16_t uTag;
3066	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3067	if (pr80Reg->s.uExponent == 0x7fff)
3068	uTag = 2; /* Exponent is all 1's => Special. */
3069	else if (pr80Reg->s.uExponent == 0x0000)
3070	{
3071	if (pr80Reg->s.uMantissa == 0x0000)
3072	uTag = 1; /* All bits are zero => Zero. */
3073	else
3074	uTag = 2; /* Must be special. */
3075	}
3076	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3077	uTag = 0; /* Valid. */
3078	else
3079	uTag = 2; /* Must be special. */
3080
3081	u16Ftw \|= uTag << (iReg * 2);
3082	}
3083	}
3084
3085	return u16Ftw;
3086	}
3087
3088
3089	/**
3090	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3091	*
3092	* @returns The compressed FTW.
3093	* @param u16FullFtw The full FTW to convert.
3094	*/
3095	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3096	{
3097	uint8_t u8Ftw = 0;
3098	for (unsigned i = 0; i < 8; i++)
3099	{
3100	if ((u16FullFtw & 3) != 3 /empty/)
3101	u8Ftw \|= RT_BIT(i);
3102	u16FullFtw >>= 2;
3103	}
3104
3105	return u8Ftw;
3106	}
3107
3108	/** @} */
3109
3110
3111	/** @name Memory access.
3112	*
3113	* @{
3114	*/
3115
3116
3117	/**
3118	* Checks whether alignment checks are enabled or not.
3119	*
3120	* @returns true if enabled, false if not.
3121	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3122	*/
3123	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3124	{
3125	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3126	return IEM_GET_CPL(pVCpu) == 3
3127	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3128	}
3129
3130	/**
3131	* Checks if the given segment can be written to, raise the appropriate
3132	* exception if not.
3133	*
3134	* @returns VBox strict status code.
3135	*
3136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3137	* @param pHid Pointer to the hidden register.
3138	* @param iSegReg The register number.
3139	* @param pu64BaseAddr Where to return the base address to use for the
3140	* segment. (In 64-bit code it may differ from the
3141	* base in the hidden segment.)
3142	*/
3143	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3144	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3145	{
3146	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3147
3148	if (IEM_IS_64BIT_CODE(pVCpu))
3149	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3150	else
3151	{
3152	if (!pHid->Attr.n.u1Present)
3153	{
3154	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3155	AssertRelease(uSel == 0);
3156	LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3157	return iemRaiseGeneralProtectionFault0(pVCpu);
3158	}
3159
3160	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3161	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3162	&& !IEM_IS_64BIT_CODE(pVCpu) )
3163	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3164	*pu64BaseAddr = pHid->u64Base;
3165	}
3166	return VINF_SUCCESS;
3167	}
3168
3169
3170	/**
3171	* Checks if the given segment can be read from, raise the appropriate
3172	* exception if not.
3173	*
3174	* @returns VBox strict status code.
3175	*
3176	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3177	* @param pHid Pointer to the hidden register.
3178	* @param iSegReg The register number.
3179	* @param pu64BaseAddr Where to return the base address to use for the
3180	* segment. (In 64-bit code it may differ from the
3181	* base in the hidden segment.)
3182	*/
3183	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3184	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3185	{
3186	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3187
3188	if (IEM_IS_64BIT_CODE(pVCpu))
3189	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3190	else
3191	{
3192	if (!pHid->Attr.n.u1Present)
3193	{
3194	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3195	AssertRelease(uSel == 0);
3196	LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3197	return iemRaiseGeneralProtectionFault0(pVCpu);
3198	}
3199
3200	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3201	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3202	*pu64BaseAddr = pHid->u64Base;
3203	}
3204	return VINF_SUCCESS;
3205	}
3206
3207
3208	/**
3209	* Maps a physical page.
3210	*
3211	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3212	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3213	* @param GCPhysMem The physical address.
3214	* @param fAccess The intended access.
3215	* @param ppvMem Where to return the mapping address.
3216	* @param pLock The PGM lock.
3217	*/
3218	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3219	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3220	{
3221	#ifdef IEM_LOG_MEMORY_WRITES
3222	if (fAccess & IEM_ACCESS_TYPE_WRITE)
3223	return VERR_PGM_PHYS_TLB_CATCH_ALL;
3224	#endif
3225
3226	/** @todo This API may require some improving later. A private deal with PGM
3227	* regarding locking and unlocking needs to be struct. A couple of TLBs
3228	* living in PGM, but with publicly accessible inlined access methods
3229	* could perhaps be an even better solution. */
3230	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3231	GCPhysMem,
3232	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3233	RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3234	ppvMem,
3235	pLock);
3236	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
3237	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3238
3239	return rc;
3240	}
3241
3242
3243	/**
3244	* Unmap a page previously mapped by iemMemPageMap.
3245	*
3246	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3247	* @param GCPhysMem The physical address.
3248	* @param fAccess The intended access.
3249	* @param pvMem What iemMemPageMap returned.
3250	* @param pLock The PGM lock.
3251	*/
3252	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3253	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3254	{
3255	NOREF(pVCpu);
3256	NOREF(GCPhysMem);
3257	NOREF(fAccess);
3258	NOREF(pvMem);
3259	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3260	}
3261
3262	#ifdef IEM_WITH_SETJMP
3263
3264	/** @todo slim this down */
3265	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3266	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3267	{
3268	Assert(cbMem >= 1);
3269	Assert(iSegReg < X86_SREG_COUNT);
3270
3271	/*
3272	* 64-bit mode is simpler.
3273	*/
3274	if (IEM_IS_64BIT_CODE(pVCpu))
3275	{
3276	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3277	{
3278	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3279	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3280	GCPtrMem += pSel->u64Base;
3281	}
3282
3283	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3284	return GCPtrMem;
3285	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3286	}
3287	/*
3288	* 16-bit and 32-bit segmentation.
3289	*/
3290	else if (iSegReg != UINT8_MAX)
3291	{
3292	/** @todo Does this apply to segments with 4G-1 limit? */
3293	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3294	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3295	{
3296	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3297	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3298	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3299	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
3300	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
3301	\| X86_SEL_TYPE_CODE))
3302	{
3303	case X86DESCATTR_P: /* readonly data, expand up */
3304	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
3305	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
3306	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
3307	/* expand up */
3308	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3309	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3310	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3311	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3312	break;
3313
3314	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3315	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
3316	/* expand down */
3317	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3318	&& ( pSel->Attr.n.u1DefBig
3319	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
3320	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3321	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3322	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3323	break;
3324
3325	default:
3326	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3327	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3328	break;
3329	}
3330	}
3331	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3332	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3333	}
3334	/*
3335	* 32-bit flat address.
3336	*/
3337	else
3338	return GCPtrMem;
3339	}
3340
3341
3342	/** @todo slim this down */
3343	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3344	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3345	{
3346	Assert(cbMem >= 1);
3347	Assert(iSegReg < X86_SREG_COUNT);
3348
3349	/*
3350	* 64-bit mode is simpler.
3351	*/
3352	if (IEM_IS_64BIT_CODE(pVCpu))
3353	{
3354	if (iSegReg >= X86_SREG_FS)
3355	{
3356	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3357	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3358	GCPtrMem += pSel->u64Base;
3359	}
3360
3361	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3362	return GCPtrMem;
3363	}
3364	/*
3365	* 16-bit and 32-bit segmentation.
3366	*/
3367	else
3368	{
3369	Assert(GCPtrMem <= UINT32_MAX);
3370	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3371	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3372	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3373	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
3374	if ( fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE) /* data, expand up */
3375	/** @todo explore exactly how the CS stuff works in real mode. See also
3376	* http://www.rcollins.org/Productivity/DescriptorCache.html and
3377	* http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */
3378	\|\| (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. / /* @todo testcase! */
3379	{
3380	/* expand up */
3381	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3382	if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit
3383	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3384	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3385	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3386	}
3387	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
3388	{
3389	/* expand down - the uppger boundary is defined by the B bit, not G. */
3390	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3391	if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit
3392	&& (pSel->Attr.n.u1DefBig \|\| GCPtrLast32 <= UINT32_C(0xffff))
3393	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3394	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3395	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3396	}
3397	else
3398	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3399	}
3400	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3401	}
3402
3403	#endif /* IEM_WITH_SETJMP */
3404
3405	/**
3406	* Fakes a long mode stack selector for SS = 0.
3407	*
3408	* @param pDescSs Where to return the fake stack descriptor.
3409	* @param uDpl The DPL we want.
3410	*/
3411	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3412	{
3413	pDescSs->Long.au64[0] = 0;
3414	pDescSs->Long.au64[1] = 0;
3415	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3416	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3417	pDescSs->Long.Gen.u2Dpl = uDpl;
3418	pDescSs->Long.Gen.u1Present = 1;
3419	pDescSs->Long.Gen.u1Long = 1;
3420	}
3421
3422
3423	/*
3424	* Unmap helpers.
3425	*/
3426
3427	#ifdef IEM_WITH_SETJMP
3428
3429	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3430	{
3431	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3432	if (RT_LIKELY(bMapInfo == 0))
3433	return;
3434	# endif
3435	iemMemCommitAndUnmapRwSafeJmp(pVCpu, pvMem, bMapInfo);
3436	}
3437
3438
3439	DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3440	{
3441	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3442	if (RT_LIKELY(bMapInfo == 0))
3443	return;
3444	# endif
3445	iemMemCommitAndUnmapWoSafeJmp(pVCpu, pvMem, bMapInfo);
3446	}
3447
3448
3449	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, const void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3450	{
3451	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3452	if (RT_LIKELY(bMapInfo == 0))
3453	return;
3454	# endif
3455	iemMemCommitAndUnmapRoSafeJmp(pVCpu, pvMem, bMapInfo);
3456	}
3457
3458	#endif /* IEM_WITH_SETJMP */
3459
3460
3461	/*
3462	* Instantiate R/W inline templates.
3463	*/
3464
3465	/** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3466	* Used to check if an unaligned access is if within the page and won't
3467	* trigger an #AC.
3468	*
3469	* This can be used to deal with misaligned accesses on platforms that are
3470	* senstive to such if desires.
3471	*/
3472	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3473	AssertCompile(((3U + 1U) << 16) == X86_CR0_AM);
3474	#if 1
3475	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \
3476	( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \
3477	&& !( (uint32_t)(a_pVCpu)->cpum.GstCtx.cr0 \
3478	& (a_pVCpu)->cpum.GstCtx.eflags.u \
3479	& ((IEM_GET_CPL((a_pVCpu)) + 1U) << 16) /* IEM_GET_CPL(a_pVCpu) == 3 ? X86_CR0_AM : 0 */ \
3480	& X86_CR0_AM) )
3481	#else
3482	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0
3483	#endif
3484
3485	#define TMPL_MEM_TYPE uint8_t
3486	#define TMPL_MEM_TYPE_ALIGN 0
3487	#define TMPL_MEM_TYPE_SIZE 1
3488	#define TMPL_MEM_FN_SUFF U8
3489	#define TMPL_MEM_FMT_TYPE "%#04x"
3490	#define TMPL_MEM_FMT_DESC "byte"
3491	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3492
3493	#define TMPL_MEM_WITH_STACK
3494
3495	#define TMPL_MEM_TYPE uint16_t
3496	#define TMPL_MEM_TYPE_ALIGN 1
3497	#define TMPL_MEM_TYPE_SIZE 2
3498	#define TMPL_MEM_FN_SUFF U16
3499	#define TMPL_MEM_FMT_TYPE "%#06x"
3500	#define TMPL_MEM_FMT_DESC "word"
3501	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3502
3503	#define TMPL_WITH_PUSH_SREG
3504	#define TMPL_MEM_TYPE uint32_t
3505	#define TMPL_MEM_TYPE_ALIGN 3
3506	#define TMPL_MEM_TYPE_SIZE 4
3507	#define TMPL_MEM_FN_SUFF U32
3508	#define TMPL_MEM_FMT_TYPE "%#010x"
3509	#define TMPL_MEM_FMT_DESC "dword"
3510	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3511	#undef TMPL_WITH_PUSH_SREG
3512
3513	#define TMPL_MEM_TYPE uint64_t
3514	#define TMPL_MEM_TYPE_ALIGN 7
3515	#define TMPL_MEM_TYPE_SIZE 8
3516	#define TMPL_MEM_FN_SUFF U64
3517	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3518	#define TMPL_MEM_FMT_DESC "qword"
3519	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3520
3521	#undef TMPL_MEM_WITH_STACK
3522
3523	#define TMPL_MEM_NO_STORE
3524	#define TMPL_MEM_NO_MAPPING
3525	#define TMPL_MEM_TYPE uint64_t
3526	#define TMPL_MEM_TYPE_ALIGN 15
3527	#define TMPL_MEM_TYPE_SIZE 8
3528	#define TMPL_MEM_FN_SUFF U64AlignedU128
3529	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3530	#define TMPL_MEM_FMT_DESC "qword"
3531	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3532
3533	#undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3534
3535	/** @} */
3536
3537
3538	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3539
3540	/**
3541	* Gets CR0 fixed-0 bits in VMX operation.
3542	*
3543	* We do this rather than fetching what we report to the guest (in
3544	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3545	* values regardless of whether unrestricted-guest feature is available on the CPU.
3546	*
3547	* @returns CR0 fixed-0 bits.
3548	* @param pVCpu The cross context virtual CPU structure.
3549	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3550	* must be returned. When @c false, the CR0 fixed-0
3551	* bits for VMX root mode is returned.
3552	*
3553	*/
3554	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3555	{
3556	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3557
3558	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3559	if ( fVmxNonRootMode
3560	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3561	return VMX_V_CR0_FIXED0_UX;
3562	return VMX_V_CR0_FIXED0;
3563	}
3564
3565
3566	/**
3567	* Sets virtual-APIC write emulation as pending.
3568	*
3569	* @param pVCpu The cross context virtual CPU structure.
3570	* @param offApic The offset in the virtual-APIC page that was written.
3571	*/
3572	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3573	{
3574	Assert(offApic < XAPIC_OFF_END + 4);
3575
3576	/*
3577	* Record the currently updated APIC offset, as we need this later for figuring
3578	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
3579	* as for supplying the exit qualification when causing an APIC-write VM-exit.
3580	*/
3581	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3582
3583	/*
3584	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3585	* virtualization or APIC-write emulation).
3586	*/
3587	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3588	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3589	}
3590
3591	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3592
3593	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 100889

Download in other formats: