IEMInline.h@ 102734

Last change on this file since 102734 was 102586, checked in by vboxsync, 12 months ago
VMM/IEM: Refactored the IEM_MC_SET_RIP_Uxx_AND_FINISH MCs in prep for native translation. [build fix] bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 140.7 KB

Line
1	/* $Id: IEMInline.h 102586 2023-12-12 13:30:02Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	/* Deal with the cases that should be treated as VINF_SUCCESS first. */
49	if ( rcStrict == VINF_IEM_YIELD_PENDING_FF
50	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */
51	\|\| rcStrict == VINF_VMX_VMEXIT
52	#endif
53	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
54	\|\| rcStrict == VINF_SVM_VMEXIT
55	#endif
56	)
57	{
58	if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
59	rcStrict = VINF_SUCCESS;
60	else
61	{
62	pVCpu->iem.s.cRetPassUpStatus++;
63	rcStrict = pVCpu->iem.s.rcPassUp;
64	}
65	}
66	else if (RT_SUCCESS(rcStrict))
67	{
68	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
69	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
70	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
71	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
72	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
73	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
74	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
75	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
76	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
77	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
78	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
79	\|\| rcStrict == VINF_EM_RAW_TO_R3
80	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
81	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
82	/* raw-mode / virt handlers only: */
83	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
84	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
85	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
86	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
87	\|\| rcStrict == VINF_SELM_SYNC_GDT
88	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
89	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
90	/* nested hw.virt codes: */
91	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
92	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
93	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
94	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
95	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
96	if (rcPassUp == VINF_SUCCESS)
97	pVCpu->iem.s.cRetInfStatuses++;
98	else if ( rcPassUp < VINF_EM_FIRST
99	\|\| rcPassUp > VINF_EM_LAST
100	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
101	{
102	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
103	pVCpu->iem.s.cRetPassUpStatus++;
104	rcStrict = rcPassUp;
105	}
106	else
107	{
108	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
109	pVCpu->iem.s.cRetInfStatuses++;
110	}
111	}
112	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
113	pVCpu->iem.s.cRetAspectNotImplemented++;
114	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
115	pVCpu->iem.s.cRetInstrNotImplemented++;
116	else
117	pVCpu->iem.s.cRetErrStatuses++;
118	}
119	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
120	{
121	pVCpu->iem.s.cRetPassUpStatus++;
122	rcStrict = pVCpu->iem.s.rcPassUp;
123	}
124
125	/* Just clear it here as well. */
126	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
127
128	return rcStrict;
129	}
130
131
132	/**
133	* Sets the pass up status.
134	*
135	* @returns VINF_SUCCESS.
136	* @param pVCpu The cross context virtual CPU structure of the
137	* calling thread.
138	* @param rcPassUp The pass up status. Must be informational.
139	* VINF_SUCCESS is not allowed.
140	*/
141	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
142	{
143	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
144
145	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
146	if (rcOldPassUp == VINF_SUCCESS)
147	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148	/* If both are EM scheduling codes, use EM priority rules. */
149	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
150	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
151	{
152	if (rcPassUp < rcOldPassUp)
153	{
154	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
155	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
156	}
157	else
158	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
159	}
160	/* Override EM scheduling with specific status code. */
161	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
162	{
163	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
164	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
165	}
166	/* Don't override specific status code, first come first served. */
167	else
168	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
169	return VINF_SUCCESS;
170	}
171
172
173	/**
174	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
175	*
176	* Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
177	* reject expand down data segments and conforming code segments.
178	*
179	* ASSUMES that the CPU is in 32-bit mode.
180	*
181	* @note Will return zero when if any of the segment register state is marked
182	* external, this must be factored into assertions checking fExec
183	* consistency.
184	*
185	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
186	* @param pVCpu The cross context virtual CPU structure of the
187	* calling thread.
188	* @sa iemCalc32BitFlatIndicatorEsDs
189	*/
190	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
191	{
192	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
193	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
194	\| pVCpu->cpum.GstCtx.cs.Attr.u
195	\| pVCpu->cpum.GstCtx.ss.Attr.u
196	\| pVCpu->cpum.GstCtx.ds.Attr.u)
197	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
198	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
199	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
200	\| (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
201	\| (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
202	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
203	== 0
204	&& ( pVCpu->cpum.GstCtx.es.u64Base
205	\| pVCpu->cpum.GstCtx.cs.u64Base
206	\| pVCpu->cpum.GstCtx.ss.u64Base
207	\| pVCpu->cpum.GstCtx.ds.u64Base)
208	== 0
209	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
210	? IEM_F_MODE_X86_32BIT_FLAT : 0;
211	}
212
213
214	/**
215	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
216	* flat already.
217	*
218	* This is used by sysenter.
219	*
220	* @note Will return zero when if any of the segment register state is marked
221	* external, this must be factored into assertions checking fExec
222	* consistency.
223	*
224	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
225	* @param pVCpu The cross context virtual CPU structure of the
226	* calling thread.
227	* @sa iemCalc32BitFlatIndicator
228	*/
229	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
230	{
231	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
232	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
233	\| pVCpu->cpum.GstCtx.ds.Attr.u)
234	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
235	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
236	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
237	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
238	== 0
239	&& ( pVCpu->cpum.GstCtx.es.u64Base
240	\| pVCpu->cpum.GstCtx.ds.u64Base)
241	== 0
242	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
243	? IEM_F_MODE_X86_32BIT_FLAT : 0;
244	}
245
246
247	/**
248	* Calculates the IEM_F_MODE_XXX and CPL flags.
249	*
250	* @returns IEM_F_MODE_XXX
251	* @param pVCpu The cross context virtual CPU structure of the
252	* calling thread.
253	*/
254	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
255	{
256	/*
257	* We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
258	* here to try get this done as efficiently as possible.
259	*/
260	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
261
262	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
263	{
264	if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
265	{
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
268	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
269	{
270	Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
271	fExec \|= IEM_F_MODE_X86_32BIT_PROT \| iemCalc32BitFlatIndicator(pVCpu);
272	}
273	else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
274	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
275	fExec \|= IEM_F_MODE_X86_64BIT;
276	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
277	fExec \|= IEM_F_MODE_X86_16BIT_PROT;
278	else
279	fExec \|= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
280	return fExec;
281	}
282	return IEM_F_MODE_X86_16BIT_PROT_V86 \| (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
283	}
284
285	/* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
286	if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
287	{
288	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
289	return IEM_F_MODE_X86_16BIT;
290	return IEM_F_MODE_X86_16BIT_PRE_386;
291	}
292
293	/* 32-bit unreal mode. */
294	return IEM_F_MODE_X86_32BIT \| iemCalc32BitFlatIndicator(pVCpu);
295	}
296
297
298	/**
299	* Calculates the AMD-V and VT-x related context flags.
300	*
301	* @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
302	* IEM_F_X86_CTX_VMX.
303	* @param pVCpu The cross context virtual CPU structure of the
304	* calling thread.
305	*/
306	DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
307	{
308	/*
309	* This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
310	* and CPUMIsGuestInNestedHwvirtMode to some extent.
311	*/
312	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
313
314	AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
315	uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
316	\| (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
317	if (RT_LIKELY(!fTmp))
318	return 0; /* likely */
319
320	if (fTmp & X86_CR4_VMXE)
321	{
322	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
323	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
324	return IEM_F_X86_CTX_VMX \| IEM_F_X86_CTX_IN_GUEST;
325	return IEM_F_X86_CTX_VMX;
326	}
327
328	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
329	if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
330	return IEM_F_X86_CTX_SVM \| IEM_F_X86_CTX_IN_GUEST;
331	return IEM_F_X86_CTX_SVM;
332	}
333
334	#ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */
335
336	/**
337	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
338	*
339	* @returns IEM_F_BRK_PENDING_XXX or zero.
340	* @param pVCpu The cross context virtual CPU structure of the
341	* calling thread.
342	*/
343	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
344	{
345	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
346
347	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
348	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
349	return 0;
350	return iemCalcExecDbgFlagsSlow(pVCpu);
351	}
352
353	/**
354	* Calculates the the IEM_F_XXX flags.
355	*
356	* @returns IEM_F_XXX combination match the current CPU state.
357	* @param pVCpu The cross context virtual CPU structure of the
358	* calling thread.
359	*/
360	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
361	{
362	return iemCalcExecModeAndCplFlags(pVCpu)
363	\| iemCalcExecHwVirtFlags(pVCpu)
364	/* SMM is not yet implemented */
365	\| iemCalcExecDbgFlags(pVCpu)
366	;
367	}
368
369
370	/**
371	* Re-calculates the MODE and CPL parts of IEMCPU::fExec.
372	*
373	* @param pVCpu The cross context virtual CPU structure of the
374	* calling thread.
375	*/
376	DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
377	{
378	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
379	\| iemCalcExecModeAndCplFlags(pVCpu);
380	}
381
382
383	/**
384	* Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
385	*
386	* @param pVCpu The cross context virtual CPU structure of the
387	* calling thread.
388	*/
389	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
390	{
391	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
392	\| iemCalcExecDbgFlags(pVCpu);
393	}
394
395	#endif /* VBOX_INCLUDED_vmm_dbgf_h */
396
397
398	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
399
400	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
401
402	/**
403	* Initializes the execution state.
404	*
405	* @param pVCpu The cross context virtual CPU structure of the
406	* calling thread.
407	* @param fExecOpts Optional execution flags:
408	* - IEM_F_BYPASS_HANDLERS
409	* - IEM_F_X86_DISREGARD_LOCK
410	*
411	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
412	* side-effects in strict builds.
413	*/
414	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
415	{
416	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
417	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
418	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
419	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
420	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
421	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
422	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
423	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
424	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
425	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
426
427	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
428	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
429	pVCpu->iem.s.cActiveMappings = 0;
430	pVCpu->iem.s.iNextMapping = 0;
431
432	# ifdef VBOX_STRICT
433	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
434	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
435	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
436	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
437	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
438	pVCpu->iem.s.uRexReg = 127;
439	pVCpu->iem.s.uRexB = 127;
440	pVCpu->iem.s.offModRm = 127;
441	pVCpu->iem.s.uRexIndex = 127;
442	pVCpu->iem.s.iEffSeg = 127;
443	pVCpu->iem.s.idxPrefix = 127;
444	pVCpu->iem.s.uVex3rdReg = 127;
445	pVCpu->iem.s.uVexLength = 127;
446	pVCpu->iem.s.fEvexStuff = 127;
447	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
448	# ifdef IEM_WITH_CODE_TLB
449	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
450	pVCpu->iem.s.pbInstrBuf = NULL;
451	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
452	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
453	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
454	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
455	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
456	pVCpu->iem.s.offOpcode = 127;
457	# endif
458	# else
459	pVCpu->iem.s.offOpcode = 127;
460	pVCpu->iem.s.cbOpcode = 127;
461	# endif
462	# endif /* VBOX_STRICT */
463	}
464
465
466	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
467	/**
468	* Performs a minimal reinitialization of the execution state.
469	*
470	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
471	* 'world-switch' types operations on the CPU. Currently only nested
472	* hardware-virtualization uses it.
473	*
474	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
475	* @param cbInstr The instruction length (for flushing).
476	*/
477	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
478	{
479	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
480	iemOpcodeFlushHeavy(pVCpu, cbInstr);
481	}
482	# endif
483
484	# endif /* VBOX_INCLUDED_vmm_dbgf_h \|\| DOXYGEN_RUNNING */
485
486	/**
487	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
488	*
489	* @param pVCpu The cross context virtual CPU structure of the
490	* calling thread.
491	*/
492	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
493	{
494	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
495	# ifdef VBOX_STRICT
496	# ifdef IEM_WITH_CODE_TLB
497	NOREF(pVCpu);
498	# else
499	pVCpu->iem.s.cbOpcode = 0;
500	# endif
501	# else
502	NOREF(pVCpu);
503	# endif
504	}
505
506
507	/**
508	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
509	*
510	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
511	*
512	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
514	* @param rcStrict The status code to fiddle.
515	*/
516	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
517	{
518	iemUninitExec(pVCpu);
519	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
520	}
521
522
523	/**
524	* Macro used by the IEMExec* method to check the given instruction length.
525	*
526	* Will return on failure!
527	*
528	* @param a_cbInstr The given instruction length.
529	* @param a_cbMin The minimum length.
530	*/
531	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
532	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
533	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
534
535
536	# ifndef IEM_WITH_SETJMP
537
538	/**
539	* Fetches the first opcode byte.
540	*
541	* @returns Strict VBox status code.
542	* @param pVCpu The cross context virtual CPU structure of the
543	* calling thread.
544	* @param pu8 Where to return the opcode byte.
545	*/
546	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
547	{
548	/*
549	* Check for hardware instruction breakpoints.
550	*/
551	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
552	{ /* likely */ }
553	else
554	{
555	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
556	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
557	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
558	{ /* likely */ }
559	else
560	{
561	pu8 = 0xff; / shut up gcc. sigh */
562	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
563	return iemRaiseDebugException(pVCpu);
564	return rcStrict;
565	}
566	}
567
568	/*
569	* Fetch the first opcode byte.
570	*/
571	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
572	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
573	{
574	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
575	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
576	return VINF_SUCCESS;
577	}
578	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
579	}
580
581	# else /* IEM_WITH_SETJMP */
582
583	/**
584	* Fetches the first opcode byte, longjmp on error.
585	*
586	* @returns The opcode byte.
587	* @param pVCpu The cross context virtual CPU structure of the calling thread.
588	*/
589	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
590	{
591	/*
592	* Check for hardware instruction breakpoints.
593	*/
594	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
595	{ /* likely */ }
596	else
597	{
598	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
599	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
600	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
601	{ /* likely */ }
602	else
603	{
604	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
605	rcStrict = iemRaiseDebugException(pVCpu);
606	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
607	}
608	}
609
610	/*
611	* Fetch the first opcode byte.
612	*/
613	# ifdef IEM_WITH_CODE_TLB
614	uint8_t bRet;
615	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
616	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
617	if (RT_LIKELY( pbBuf != NULL
618	&& offBuf < pVCpu->iem.s.cbInstrBuf))
619	{
620	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
621	bRet = pbBuf[offBuf];
622	}
623	else
624	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
625	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
626	Assert(pVCpu->iem.s.offOpcode == 0);
627	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
628	# endif
629	return bRet;
630
631	# else /* !IEM_WITH_CODE_TLB */
632	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
633	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
634	{
635	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
636	return pVCpu->iem.s.abOpcode[offOpcode];
637	}
638	return iemOpcodeGetNextU8SlowJmp(pVCpu);
639	# endif
640	}
641
642	# endif /* IEM_WITH_SETJMP */
643
644	/**
645	* Fetches the first opcode byte, returns/throws automatically on failure.
646	*
647	* @param a_pu8 Where to return the opcode byte.
648	* @remark Implicitly references pVCpu.
649	*/
650	# ifndef IEM_WITH_SETJMP
651	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
652	do \
653	{ \
654	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
655	if (rcStrict2 == VINF_SUCCESS) \
656	{ /* likely */ } \
657	else \
658	return rcStrict2; \
659	} while (0)
660	# else
661	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
662	# endif /* IEM_WITH_SETJMP */
663
664
665	# ifndef IEM_WITH_SETJMP
666
667	/**
668	* Fetches the next opcode byte.
669	*
670	* @returns Strict VBox status code.
671	* @param pVCpu The cross context virtual CPU structure of the
672	* calling thread.
673	* @param pu8 Where to return the opcode byte.
674	*/
675	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
676	{
677	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
678	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
679	{
680	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
681	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
682	return VINF_SUCCESS;
683	}
684	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
685	}
686
687	# else /* IEM_WITH_SETJMP */
688
689	/**
690	* Fetches the next opcode byte, longjmp on error.
691	*
692	* @returns The opcode byte.
693	* @param pVCpu The cross context virtual CPU structure of the calling thread.
694	*/
695	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
696	{
697	# ifdef IEM_WITH_CODE_TLB
698	uint8_t bRet;
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf < pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
705	bRet = pbBuf[offBuf];
706	}
707	else
708	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
709	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
710	Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
711	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
712	# endif
713	return bRet;
714
715	# else /* !IEM_WITH_CODE_TLB */
716	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
717	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
718	{
719	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
720	return pVCpu->iem.s.abOpcode[offOpcode];
721	}
722	return iemOpcodeGetNextU8SlowJmp(pVCpu);
723	# endif
724	}
725
726	# endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode byte, returns automatically on failure.
730	*
731	* @param a_pu8 Where to return the opcode byte.
732	* @remark Implicitly references pVCpu.
733	*/
734	# ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
739	if (rcStrict2 == VINF_SUCCESS) \
740	{ /* likely */ } \
741	else \
742	return rcStrict2; \
743	} while (0)
744	# else
745	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
746	# endif /* IEM_WITH_SETJMP */
747
748
749	# ifndef IEM_WITH_SETJMP
750	/**
751	* Fetches the next signed byte from the opcode stream.
752	*
753	* @returns Strict VBox status code.
754	* @param pVCpu The cross context virtual CPU structure of the calling thread.
755	* @param pi8 Where to return the signed byte.
756	*/
757	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
758	{
759	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
760	}
761	# endif /* !IEM_WITH_SETJMP */
762
763
764	/**
765	* Fetches the next signed byte from the opcode stream, returning automatically
766	* on failure.
767	*
768	* @param a_pi8 Where to return the signed byte.
769	* @remark Implicitly references pVCpu.
770	*/
771	# ifndef IEM_WITH_SETJMP
772	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
773	do \
774	{ \
775	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
776	if (rcStrict2 != VINF_SUCCESS) \
777	return rcStrict2; \
778	} while (0)
779	# else /* IEM_WITH_SETJMP */
780	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
781
782	# endif /* IEM_WITH_SETJMP */
783
784
785	# ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next signed byte from the opcode stream, extending it to
788	* unsigned 16-bit.
789	*
790	* @returns Strict VBox status code.
791	* @param pVCpu The cross context virtual CPU structure of the calling thread.
792	* @param pu16 Where to return the unsigned word.
793	*/
794	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
795	{
796	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
797	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
798	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
799
800	*pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
801	pVCpu->iem.s.offOpcode = offOpcode + 1;
802	return VINF_SUCCESS;
803	}
804	# endif /* !IEM_WITH_SETJMP */
805
806	/**
807	* Fetches the next signed byte from the opcode stream and sign-extending it to
808	* a word, returning automatically on failure.
809	*
810	* @param a_pu16 Where to return the word.
811	* @remark Implicitly references pVCpu.
812	*/
813	# ifndef IEM_WITH_SETJMP
814	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
815	do \
816	{ \
817	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
818	if (rcStrict2 != VINF_SUCCESS) \
819	return rcStrict2; \
820	} while (0)
821	# else
822	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
823	# endif
824
825	# ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed byte from the opcode stream, extending it to
828	* unsigned 32-bit.
829	*
830	* @returns Strict VBox status code.
831	* @param pVCpu The cross context virtual CPU structure of the calling thread.
832	* @param pu32 Where to return the unsigned dword.
833	*/
834	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
835	{
836	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
837	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
838	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
839
840	*pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
841	pVCpu->iem.s.offOpcode = offOpcode + 1;
842	return VINF_SUCCESS;
843	}
844	# endif /* !IEM_WITH_SETJMP */
845
846	/**
847	* Fetches the next signed byte from the opcode stream and sign-extending it to
848	* a word, returning automatically on failure.
849	*
850	* @param a_pu32 Where to return the word.
851	* @remark Implicitly references pVCpu.
852	*/
853	# ifndef IEM_WITH_SETJMP
854	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
855	do \
856	{ \
857	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
858	if (rcStrict2 != VINF_SUCCESS) \
859	return rcStrict2; \
860	} while (0)
861	# else
862	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
863	# endif
864
865
866	# ifndef IEM_WITH_SETJMP
867	/**
868	* Fetches the next signed byte from the opcode stream, extending it to
869	* unsigned 64-bit.
870	*
871	* @returns Strict VBox status code.
872	* @param pVCpu The cross context virtual CPU structure of the calling thread.
873	* @param pu64 Where to return the unsigned qword.
874	*/
875	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
876	{
877	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
878	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
879	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
880
881	*pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
882	pVCpu->iem.s.offOpcode = offOpcode + 1;
883	return VINF_SUCCESS;
884	}
885	# endif /* !IEM_WITH_SETJMP */
886
887	/**
888	* Fetches the next signed byte from the opcode stream and sign-extending it to
889	* a word, returning automatically on failure.
890	*
891	* @param a_pu64 Where to return the word.
892	* @remark Implicitly references pVCpu.
893	*/
894	# ifndef IEM_WITH_SETJMP
895	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
896	do \
897	{ \
898	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
899	if (rcStrict2 != VINF_SUCCESS) \
900	return rcStrict2; \
901	} while (0)
902	# else
903	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
904	# endif
905
906
907	# ifndef IEM_WITH_SETJMP
908
909	/**
910	* Fetches the next opcode word.
911	*
912	* @returns Strict VBox status code.
913	* @param pVCpu The cross context virtual CPU structure of the calling thread.
914	* @param pu16 Where to return the opcode word.
915	*/
916	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
917	{
918	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
919	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
920	{
921	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
922	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
923	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
924	# else
925	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
926	# endif
927	return VINF_SUCCESS;
928	}
929	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
930	}
931
932	# else /* IEM_WITH_SETJMP */
933
934	/**
935	* Fetches the next opcode word, longjmp on error.
936	*
937	* @returns The opcode word.
938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
939	*/
940	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
941	{
942	# ifdef IEM_WITH_CODE_TLB
943	uint16_t u16Ret;
944	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
945	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
946	if (RT_LIKELY( pbBuf != NULL
947	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
948	{
949	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
950	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
951	u16Ret = (uint16_t const )&pbBuf[offBuf];
952	# else
953	u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
954	# endif
955	}
956	else
957	u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
958
959	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
960	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
961	Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
962	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
963	(uint16_t )&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
964	# else
965	pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
966	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
967	# endif
968	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
969	# endif
970
971	return u16Ret;
972
973	# else /* !IEM_WITH_CODE_TLB */
974	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
975	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
976	{
977	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
978	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
979	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
980	# else
981	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
982	# endif
983	}
984	return iemOpcodeGetNextU16SlowJmp(pVCpu);
985	# endif /* !IEM_WITH_CODE_TLB */
986	}
987
988	# endif /* IEM_WITH_SETJMP */
989
990	/**
991	* Fetches the next opcode word, returns automatically on failure.
992	*
993	* @param a_pu16 Where to return the opcode word.
994	* @remark Implicitly references pVCpu.
995	*/
996	# ifndef IEM_WITH_SETJMP
997	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
998	do \
999	{ \
1000	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
1001	if (rcStrict2 != VINF_SUCCESS) \
1002	return rcStrict2; \
1003	} while (0)
1004	# else
1005	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
1006	# endif
1007
1008	# ifndef IEM_WITH_SETJMP
1009	/**
1010	* Fetches the next opcode word, zero extending it to a double word.
1011	*
1012	* @returns Strict VBox status code.
1013	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1014	* @param pu32 Where to return the opcode double word.
1015	*/
1016	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1017	{
1018	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1019	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1020	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1021
1022	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1023	pVCpu->iem.s.offOpcode = offOpcode + 2;
1024	return VINF_SUCCESS;
1025	}
1026	# endif /* !IEM_WITH_SETJMP */
1027
1028	/**
1029	* Fetches the next opcode word and zero extends it to a double word, returns
1030	* automatically on failure.
1031	*
1032	* @param a_pu32 Where to return the opcode double word.
1033	* @remark Implicitly references pVCpu.
1034	*/
1035	# ifndef IEM_WITH_SETJMP
1036	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1037	do \
1038	{ \
1039	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1040	if (rcStrict2 != VINF_SUCCESS) \
1041	return rcStrict2; \
1042	} while (0)
1043	# else
1044	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1045	# endif
1046
1047	# ifndef IEM_WITH_SETJMP
1048	/**
1049	* Fetches the next opcode word, zero extending it to a quad word.
1050	*
1051	* @returns Strict VBox status code.
1052	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1053	* @param pu64 Where to return the opcode quad word.
1054	*/
1055	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1056	{
1057	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1058	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1059	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1060
1061	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1062	pVCpu->iem.s.offOpcode = offOpcode + 2;
1063	return VINF_SUCCESS;
1064	}
1065	# endif /* !IEM_WITH_SETJMP */
1066
1067	/**
1068	* Fetches the next opcode word and zero extends it to a quad word, returns
1069	* automatically on failure.
1070	*
1071	* @param a_pu64 Where to return the opcode quad word.
1072	* @remark Implicitly references pVCpu.
1073	*/
1074	# ifndef IEM_WITH_SETJMP
1075	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1076	do \
1077	{ \
1078	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1079	if (rcStrict2 != VINF_SUCCESS) \
1080	return rcStrict2; \
1081	} while (0)
1082	# else
1083	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1084	# endif
1085
1086
1087	# ifndef IEM_WITH_SETJMP
1088	/**
1089	* Fetches the next signed word from the opcode stream.
1090	*
1091	* @returns Strict VBox status code.
1092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1093	* @param pi16 Where to return the signed word.
1094	*/
1095	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1096	{
1097	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1098	}
1099	# endif /* !IEM_WITH_SETJMP */
1100
1101
1102	/**
1103	* Fetches the next signed word from the opcode stream, returning automatically
1104	* on failure.
1105	*
1106	* @param a_pi16 Where to return the signed word.
1107	* @remark Implicitly references pVCpu.
1108	*/
1109	# ifndef IEM_WITH_SETJMP
1110	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1111	do \
1112	{ \
1113	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1114	if (rcStrict2 != VINF_SUCCESS) \
1115	return rcStrict2; \
1116	} while (0)
1117	# else
1118	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1119	# endif
1120
1121	# ifndef IEM_WITH_SETJMP
1122
1123	/**
1124	* Fetches the next opcode dword.
1125	*
1126	* @returns Strict VBox status code.
1127	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1128	* @param pu32 Where to return the opcode double word.
1129	*/
1130	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1131	{
1132	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1133	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1134	{
1135	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1136	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1137	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1138	# else
1139	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1140	pVCpu->iem.s.abOpcode[offOpcode + 1],
1141	pVCpu->iem.s.abOpcode[offOpcode + 2],
1142	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1143	# endif
1144	return VINF_SUCCESS;
1145	}
1146	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1147	}
1148
1149	# else /* IEM_WITH_SETJMP */
1150
1151	/**
1152	* Fetches the next opcode dword, longjmp on error.
1153	*
1154	* @returns The opcode dword.
1155	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1156	*/
1157	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1158	{
1159	# ifdef IEM_WITH_CODE_TLB
1160	uint32_t u32Ret;
1161	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1162	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1163	if (RT_LIKELY( pbBuf != NULL
1164	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1165	{
1166	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1167	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1168	u32Ret = (uint32_t const )&pbBuf[offBuf];
1169	# else
1170	u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1171	pbBuf[offBuf + 1],
1172	pbBuf[offBuf + 2],
1173	pbBuf[offBuf + 3]);
1174	# endif
1175	}
1176	else
1177	u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1178
1179	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1180	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1181	Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1182	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1183	(uint32_t )&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1184	# else
1185	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1186	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1187	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1188	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1189	# endif
1190	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1191	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1192
1193	return u32Ret;
1194
1195	# else /* !IEM_WITH_CODE_TLB */
1196	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1197	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1198	{
1199	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1200	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1201	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1202	# else
1203	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1204	pVCpu->iem.s.abOpcode[offOpcode + 1],
1205	pVCpu->iem.s.abOpcode[offOpcode + 2],
1206	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1207	# endif
1208	}
1209	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1210	# endif
1211	}
1212
1213	# endif /* IEM_WITH_SETJMP */
1214
1215	/**
1216	* Fetches the next opcode dword, returns automatically on failure.
1217	*
1218	* @param a_pu32 Where to return the opcode dword.
1219	* @remark Implicitly references pVCpu.
1220	*/
1221	# ifndef IEM_WITH_SETJMP
1222	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1223	do \
1224	{ \
1225	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1226	if (rcStrict2 != VINF_SUCCESS) \
1227	return rcStrict2; \
1228	} while (0)
1229	# else
1230	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1231	# endif
1232
1233	# ifndef IEM_WITH_SETJMP
1234	/**
1235	* Fetches the next opcode dword, zero extending it to a quad word.
1236	*
1237	* @returns Strict VBox status code.
1238	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1239	* @param pu64 Where to return the opcode quad word.
1240	*/
1241	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1242	{
1243	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1244	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1245	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1246
1247	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1248	pVCpu->iem.s.abOpcode[offOpcode + 1],
1249	pVCpu->iem.s.abOpcode[offOpcode + 2],
1250	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1251	pVCpu->iem.s.offOpcode = offOpcode + 4;
1252	return VINF_SUCCESS;
1253	}
1254	# endif /* !IEM_WITH_SETJMP */
1255
1256	/**
1257	* Fetches the next opcode dword and zero extends it to a quad word, returns
1258	* automatically on failure.
1259	*
1260	* @param a_pu64 Where to return the opcode quad word.
1261	* @remark Implicitly references pVCpu.
1262	*/
1263	# ifndef IEM_WITH_SETJMP
1264	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1265	do \
1266	{ \
1267	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1268	if (rcStrict2 != VINF_SUCCESS) \
1269	return rcStrict2; \
1270	} while (0)
1271	# else
1272	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1273	# endif
1274
1275
1276	# ifndef IEM_WITH_SETJMP
1277	/**
1278	* Fetches the next signed double word from the opcode stream.
1279	*
1280	* @returns Strict VBox status code.
1281	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1282	* @param pi32 Where to return the signed double word.
1283	*/
1284	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1285	{
1286	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1287	}
1288	# endif
1289
1290	/**
1291	* Fetches the next signed double word from the opcode stream, returning
1292	* automatically on failure.
1293	*
1294	* @param a_pi32 Where to return the signed double word.
1295	* @remark Implicitly references pVCpu.
1296	*/
1297	# ifndef IEM_WITH_SETJMP
1298	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1299	do \
1300	{ \
1301	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1302	if (rcStrict2 != VINF_SUCCESS) \
1303	return rcStrict2; \
1304	} while (0)
1305	# else
1306	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1307	# endif
1308
1309	# ifndef IEM_WITH_SETJMP
1310	/**
1311	* Fetches the next opcode dword, sign extending it into a quad word.
1312	*
1313	* @returns Strict VBox status code.
1314	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1315	* @param pu64 Where to return the opcode quad word.
1316	*/
1317	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1318	{
1319	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1320	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1321	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1322
1323	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1324	pVCpu->iem.s.abOpcode[offOpcode + 1],
1325	pVCpu->iem.s.abOpcode[offOpcode + 2],
1326	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1327	*pu64 = (uint64_t)(int64_t)i32;
1328	pVCpu->iem.s.offOpcode = offOpcode + 4;
1329	return VINF_SUCCESS;
1330	}
1331	# endif /* !IEM_WITH_SETJMP */
1332
1333	/**
1334	* Fetches the next opcode double word and sign extends it to a quad word,
1335	* returns automatically on failure.
1336	*
1337	* @param a_pu64 Where to return the opcode quad word.
1338	* @remark Implicitly references pVCpu.
1339	*/
1340	# ifndef IEM_WITH_SETJMP
1341	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1342	do \
1343	{ \
1344	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1345	if (rcStrict2 != VINF_SUCCESS) \
1346	return rcStrict2; \
1347	} while (0)
1348	# else
1349	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1350	# endif
1351
1352	# ifndef IEM_WITH_SETJMP
1353
1354	/**
1355	* Fetches the next opcode qword.
1356	*
1357	* @returns Strict VBox status code.
1358	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1359	* @param pu64 Where to return the opcode qword.
1360	*/
1361	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1362	{
1363	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1364	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1365	{
1366	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1367	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1368	# else
1369	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1370	pVCpu->iem.s.abOpcode[offOpcode + 1],
1371	pVCpu->iem.s.abOpcode[offOpcode + 2],
1372	pVCpu->iem.s.abOpcode[offOpcode + 3],
1373	pVCpu->iem.s.abOpcode[offOpcode + 4],
1374	pVCpu->iem.s.abOpcode[offOpcode + 5],
1375	pVCpu->iem.s.abOpcode[offOpcode + 6],
1376	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1377	# endif
1378	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1379	return VINF_SUCCESS;
1380	}
1381	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1382	}
1383
1384	# else /* IEM_WITH_SETJMP */
1385
1386	/**
1387	* Fetches the next opcode qword, longjmp on error.
1388	*
1389	* @returns The opcode qword.
1390	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1391	*/
1392	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1393	{
1394	# ifdef IEM_WITH_CODE_TLB
1395	uint64_t u64Ret;
1396	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1397	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1398	if (RT_LIKELY( pbBuf != NULL
1399	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1400	{
1401	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1402	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1403	u64Ret = (uint64_t const )&pbBuf[offBuf];
1404	# else
1405	u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1406	pbBuf[offBuf + 1],
1407	pbBuf[offBuf + 2],
1408	pbBuf[offBuf + 3],
1409	pbBuf[offBuf + 4],
1410	pbBuf[offBuf + 5],
1411	pbBuf[offBuf + 6],
1412	pbBuf[offBuf + 7]);
1413	# endif
1414	}
1415	else
1416	u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1417
1418	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1419	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1420	Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1421	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1422	(uint64_t )&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1423	# else
1424	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1425	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1426	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1427	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1428	pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1429	pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1430	pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1431	pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1432	# endif
1433	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1434	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1435
1436	return u64Ret;
1437
1438	# else /* !IEM_WITH_CODE_TLB */
1439	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1440	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1441	{
1442	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1443	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1444	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1445	# else
1446	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1447	pVCpu->iem.s.abOpcode[offOpcode + 1],
1448	pVCpu->iem.s.abOpcode[offOpcode + 2],
1449	pVCpu->iem.s.abOpcode[offOpcode + 3],
1450	pVCpu->iem.s.abOpcode[offOpcode + 4],
1451	pVCpu->iem.s.abOpcode[offOpcode + 5],
1452	pVCpu->iem.s.abOpcode[offOpcode + 6],
1453	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1454	# endif
1455	}
1456	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1457	# endif /* !IEM_WITH_CODE_TLB */
1458	}
1459
1460	# endif /* IEM_WITH_SETJMP */
1461
1462	/**
1463	* Fetches the next opcode quad word, returns automatically on failure.
1464	*
1465	* @param a_pu64 Where to return the opcode quad word.
1466	* @remark Implicitly references pVCpu.
1467	*/
1468	# ifndef IEM_WITH_SETJMP
1469	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1470	do \
1471	{ \
1472	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1473	if (rcStrict2 != VINF_SUCCESS) \
1474	return rcStrict2; \
1475	} while (0)
1476	# else
1477	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1478	# endif
1479
1480	/**
1481	* For fetching the opcode bytes for an ModR/M effective address, but throw
1482	* away the result.
1483	*
1484	* This is used when decoding undefined opcodes and such where we want to avoid
1485	* unnecessary MC blocks.
1486	*
1487	* @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1488	* used instead. At least for now...
1489	*/
1490	# ifndef IEM_WITH_SETJMP
1491	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1492	RTGCPTR GCPtrEff; \
1493	VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1494	if (rcStrict != VINF_SUCCESS) \
1495	return rcStrict; \
1496	} while (0)
1497	# else
1498	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1499	(void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1500	} while (0)
1501	# endif
1502
1503	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1504
1505
1506	/** @name Misc Worker Functions.
1507	* @{
1508	*/
1509
1510	/**
1511	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1512	* not (kind of obsolete now).
1513	*
1514	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1515	*/
1516	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1517
1518	/**
1519	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1520	*
1521	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1522	* @param a_fEfl The new EFLAGS.
1523	*/
1524	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1525
1526
1527	/**
1528	* Loads a NULL data selector into a selector register, both the hidden and
1529	* visible parts, in protected mode.
1530	*
1531	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1532	* @param pSReg Pointer to the segment register.
1533	* @param uRpl The RPL.
1534	*/
1535	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1536	{
1537	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1538	* data selector in protected mode. */
1539	pSReg->Sel = uRpl;
1540	pSReg->ValidSel = uRpl;
1541	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1542	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1543	{
1544	/* VT-x (Intel 3960x) observed doing something like this. */
1545	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1546	pSReg->u32Limit = UINT32_MAX;
1547	pSReg->u64Base = 0;
1548	}
1549	else
1550	{
1551	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1552	pSReg->u32Limit = 0;
1553	pSReg->u64Base = 0;
1554	}
1555	}
1556
1557	/** @} */
1558
1559
1560	/*
1561	*
1562	* Helpers routines.
1563	* Helpers routines.
1564	* Helpers routines.
1565	*
1566	*/
1567
1568	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1569
1570	/**
1571	* Recalculates the effective operand size.
1572	*
1573	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1574	*/
1575	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1576	{
1577	switch (IEM_GET_CPU_MODE(pVCpu))
1578	{
1579	case IEMMODE_16BIT:
1580	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1581	break;
1582	case IEMMODE_32BIT:
1583	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1584	break;
1585	case IEMMODE_64BIT:
1586	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1587	{
1588	case 0:
1589	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1590	break;
1591	case IEM_OP_PRF_SIZE_OP:
1592	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1593	break;
1594	case IEM_OP_PRF_SIZE_REX_W:
1595	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1596	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1597	break;
1598	}
1599	break;
1600	default:
1601	AssertFailed();
1602	}
1603	}
1604
1605
1606	/**
1607	* Sets the default operand size to 64-bit and recalculates the effective
1608	* operand size.
1609	*
1610	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1611	*/
1612	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1613	{
1614	Assert(IEM_IS_64BIT_CODE(pVCpu));
1615	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1616	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1617	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1618	else
1619	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1620	}
1621
1622
1623	/**
1624	* Sets the default operand size to 64-bit and recalculates the effective
1625	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1626	*
1627	* This is for the relative jumps.
1628	*
1629	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1630	*/
1631	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1632	{
1633	Assert(IEM_IS_64BIT_CODE(pVCpu));
1634	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1635	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1636	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1637	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1638	else
1639	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1640	}
1641
1642	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1643
1644
1645
1646	/** @name Register Access.
1647	* @{
1648	*/
1649
1650	/**
1651	* Gets a reference (pointer) to the specified hidden segment register.
1652	*
1653	* @returns Hidden register reference.
1654	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1655	* @param iSegReg The segment register.
1656	*/
1657	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1658	{
1659	Assert(iSegReg < X86_SREG_COUNT);
1660	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1661	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1662
1663	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1664	return pSReg;
1665	}
1666
1667
1668	/**
1669	* Ensures that the given hidden segment register is up to date.
1670	*
1671	* @returns Hidden register reference.
1672	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1673	* @param pSReg The segment register.
1674	*/
1675	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1676	{
1677	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1678	NOREF(pVCpu);
1679	return pSReg;
1680	}
1681
1682
1683	/**
1684	* Gets a reference (pointer) to the specified segment register (the selector
1685	* value).
1686	*
1687	* @returns Pointer to the selector variable.
1688	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1689	* @param iSegReg The segment register.
1690	*/
1691	DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1692	{
1693	Assert(iSegReg < X86_SREG_COUNT);
1694	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1695	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1696	}
1697
1698
1699	/**
1700	* Fetches the selector value of a segment register.
1701	*
1702	* @returns The selector value.
1703	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1704	* @param iSegReg The segment register.
1705	*/
1706	DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1707	{
1708	Assert(iSegReg < X86_SREG_COUNT);
1709	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1710	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1711	}
1712
1713
1714	/**
1715	* Fetches the base address value of a segment register.
1716	*
1717	* @returns The selector value.
1718	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1719	* @param iSegReg The segment register.
1720	*/
1721	DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1722	{
1723	Assert(iSegReg < X86_SREG_COUNT);
1724	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1725	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1726	}
1727
1728
1729	/**
1730	* Gets a reference (pointer) to the specified general purpose register.
1731	*
1732	* @returns Register reference.
1733	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1734	* @param iReg The general purpose register.
1735	*/
1736	DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1737	{
1738	Assert(iReg < 16);
1739	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1740	}
1741
1742
1743	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1744	/**
1745	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1746	*
1747	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1748	*
1749	* @returns Register reference.
1750	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1751	* @param iReg The register.
1752	*/
1753	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1754	{
1755	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1756	{
1757	Assert(iReg < 16);
1758	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1759	}
1760	/* high 8-bit register. */
1761	Assert(iReg < 8);
1762	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1763	}
1764	#endif
1765
1766
1767	/**
1768	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1769	* alternative version with extended (20) register index.
1770	*
1771	* @returns Register reference.
1772	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1773	* @param iRegEx The register. The 16 first are regular ones,
1774	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1775	*/
1776	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1777	{
1778	/** @todo This could be done by double indexing on little endian hosts:
1779	* return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1780	if (iRegEx < 16)
1781	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1782
1783	/* high 8-bit register. */
1784	Assert(iRegEx < 20);
1785	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1786	}
1787
1788
1789	/**
1790	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1791	*
1792	* @returns Register reference.
1793	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1794	* @param iReg The register.
1795	*/
1796	DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1797	{
1798	Assert(iReg < 16);
1799	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1800	}
1801
1802
1803	/**
1804	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1805	*
1806	* @returns Register reference.
1807	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1808	* @param iReg The register.
1809	*/
1810	DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1811	{
1812	Assert(iReg < 16);
1813	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1814	}
1815
1816
1817	/**
1818	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1819	*
1820	* @returns Register reference.
1821	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1822	* @param iReg The register.
1823	*/
1824	DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1825	{
1826	Assert(iReg < 16);
1827	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1828	}
1829
1830
1831	/**
1832	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1833	*
1834	* @returns Register reference.
1835	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1836	* @param iReg The register.
1837	*/
1838	DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1839	{
1840	Assert(iReg < 64);
1841	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1842	}
1843
1844
1845	/**
1846	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1847	*
1848	* @returns Register reference.
1849	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1850	* @param iReg The register.
1851	*/
1852	DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1853	{
1854	Assert(iReg < 16);
1855	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1856	}
1857
1858
1859	/**
1860	* Gets a reference (pointer) to the specified segment register's base address.
1861	*
1862	* @returns Segment register base address reference.
1863	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1864	* @param iSegReg The segment selector.
1865	*/
1866	DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1867	{
1868	Assert(iSegReg < X86_SREG_COUNT);
1869	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1870	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1871	}
1872
1873
1874	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1875	/**
1876	* Fetches the value of a 8-bit general purpose register.
1877	*
1878	* @returns The register value.
1879	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1880	* @param iReg The register.
1881	*/
1882	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1883	{
1884	return *iemGRegRefU8(pVCpu, iReg);
1885	}
1886	#endif
1887
1888
1889	/**
1890	* Fetches the value of a 8-bit general purpose register, alternative version
1891	* with extended (20) register index.
1892
1893	* @returns The register value.
1894	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1895	* @param iRegEx The register. The 16 first are regular ones,
1896	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1897	*/
1898	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1899	{
1900	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1901	}
1902
1903
1904	/**
1905	* Fetches the value of a 16-bit general purpose register.
1906	*
1907	* @returns The register value.
1908	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1909	* @param iReg The register.
1910	*/
1911	DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1912	{
1913	Assert(iReg < 16);
1914	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1915	}
1916
1917
1918	/**
1919	* Fetches the value of a 32-bit general purpose register.
1920	*
1921	* @returns The register value.
1922	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1923	* @param iReg The register.
1924	*/
1925	DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1926	{
1927	Assert(iReg < 16);
1928	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1929	}
1930
1931
1932	/**
1933	* Fetches the value of a 64-bit general purpose register.
1934	*
1935	* @returns The register value.
1936	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1937	* @param iReg The register.
1938	*/
1939	DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1940	{
1941	Assert(iReg < 16);
1942	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1943	}
1944
1945
1946	/**
1947	* Stores a 16-bit value to a general purpose register.
1948	*
1949	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1950	* @param iReg The register.
1951	* @param uValue The value to store.
1952	*/
1953	DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1954	{
1955	Assert(iReg < 16);
1956	pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1957	}
1958
1959
1960	/**
1961	* Stores a 32-bit value to a general purpose register, implicitly clearing high
1962	* values.
1963	*
1964	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1965	* @param iReg The register.
1966	* @param uValue The value to store.
1967	*/
1968	DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1969	{
1970	Assert(iReg < 16);
1971	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1972	}
1973
1974
1975	/**
1976	* Stores a 64-bit value to a general purpose register.
1977	*
1978	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1979	* @param iReg The register.
1980	* @param uValue The value to store.
1981	*/
1982	DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1983	{
1984	Assert(iReg < 16);
1985	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1986	}
1987
1988
1989	/**
1990	* Get the address of the top of the stack.
1991	*
1992	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1993	*/
1994	DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1995	{
1996	if (IEM_IS_64BIT_CODE(pVCpu))
1997	return pVCpu->cpum.GstCtx.rsp;
1998	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1999	return pVCpu->cpum.GstCtx.esp;
2000	return pVCpu->cpum.GstCtx.sp;
2001	}
2002
2003
2004	/**
2005	* Updates the RIP/EIP/IP to point to the next instruction.
2006	*
2007	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2008	* @param cbInstr The number of bytes to add.
2009	*/
2010	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2011	{
2012	/*
2013	* Advance RIP.
2014	*
2015	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
2016	* while in all other modes except LM64 the updates are 32-bit. This means
2017	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
2018	* 4GB and 64KB rollovers, and decide whether anything needs masking.
2019	*
2020	* See PC wrap around tests in bs3-cpu-weird-1.
2021	*/
2022	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2023	uint64_t const uRipNext = uRipPrev + cbInstr;
2024	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
2025	\|\| IEM_IS_64BIT_CODE(pVCpu)))
2026	pVCpu->cpum.GstCtx.rip = uRipNext;
2027	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2028	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2029	else
2030	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2031	}
2032
2033
2034	/**
2035	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2036	* following EFLAGS bits are set:
2037	* - X86_EFL_RF - clear it.
2038	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2039	* - X86_EFL_TF - generate single step \#DB trap.
2040	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2041	* instruction).
2042	*
2043	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2044	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2045	* takes priority over both NMIs and hardware interrupts. So, neither is
2046	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2047	* either unsupported will be triggered on-top of any \#DB raised here.)
2048	*
2049	* The RF flag only needs to be cleared here as it only suppresses instruction
2050	* breakpoints which are not raised here (happens synchronously during
2051	* instruction fetching).
2052	*
2053	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2054	* status has no bearing on whether \#DB exceptions are raised.
2055	*
2056	* @note This must NOT be called by the two instructions setting the
2057	* CPUMCTX_INHIBIT_SHADOW_SS flag.
2058	*
2059	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2060	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2061	* Stacks}
2062	*/
2063	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2064	{
2065	/*
2066	* Normally we're just here to clear RF and/or interrupt shadow bits.
2067	*/
2068	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
2069	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
2070	else
2071	{
2072	/*
2073	* Raise a #DB or/and DBGF event.
2074	*/
2075	VBOXSTRICTRC rcStrict;
2076	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
2077	{
2078	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2079	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2080	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2081	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
2082	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2083	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2084	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2085	pVCpu->cpum.GstCtx.rflags.uBoth));
2086
2087	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
2088	rcStrict = iemRaiseDebugException(pVCpu);
2089
2090	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2091	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2092	{
2093	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2094	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2095	}
2096	}
2097	else
2098	{
2099	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2100	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2101	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2102	}
2103	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2104	return rcStrict;
2105	}
2106	return VINF_SUCCESS;
2107	}
2108
2109
2110	/**
2111	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2112	*
2113	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2114	*/
2115	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2116	{
2117	/*
2118	* We assume that most of the time nothing actually needs doing here.
2119	*/
2120	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2121	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2122	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
2123	return VINF_SUCCESS;
2124	return iemFinishInstructionWithFlagsSet(pVCpu);
2125	}
2126
2127
2128	/**
2129	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2130	* and CPUMCTX_INHIBIT_SHADOW.
2131	*
2132	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2133	* @param cbInstr The number of bytes to add.
2134	*/
2135	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2136	{
2137	iemRegAddToRip(pVCpu, cbInstr);
2138	return iemRegFinishClearingRF(pVCpu);
2139	}
2140
2141
2142	/**
2143	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
2144	* and CPUMCTX_INHIBIT_SHADOW.
2145	*
2146	* Only called from 64-bit code.
2147	*
2148	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2149	* @param cbInstr The number of bytes to add.
2150	*/
2151	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2152	{
2153	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2154	return iemRegFinishClearingRF(pVCpu);
2155	}
2156
2157
2158	/**
2159	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2160	* CPUMCTX_INHIBIT_SHADOW.
2161	*
2162	* This is never from 64-bit code.
2163	*
2164	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2165	* @param cbInstr The number of bytes to add.
2166	*/
2167	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2168	{
2169	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2170	return iemRegFinishClearingRF(pVCpu);
2171	}
2172
2173
2174	/**
2175	* Updates the IP to point to the next instruction and clears EFLAGS.RF and
2176	* CPUMCTX_INHIBIT_SHADOW.
2177	*
2178	* This is only ever used from 16-bit code on a pre-386 CPU.
2179	*
2180	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2181	* @param cbInstr The number of bytes to add.
2182	*/
2183	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2184	{
2185	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2186	return iemRegFinishClearingRF(pVCpu);
2187	}
2188
2189
2190	/**
2191	* Tail method for a finish function that does't clear flags or raise \#DB.
2192	*
2193	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2194	*/
2195	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishNoFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
2196	{
2197	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2198	Assert(!( pVCpu->cpum.GstCtx.eflags.uBoth
2199	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) );
2200	RT_NOREF(pVCpu);
2201	return VINF_SUCCESS;
2202	}
2203
2204
2205	/**
2206	* Updates the RIP to point to the next instruction, but does not need to clear
2207	* EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.
2208	*
2209	* Only called from 64-bit code.
2210	*
2211	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2212	* @param cbInstr The number of bytes to add.
2213	*/
2214	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2215	{
2216	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2217	return iemRegFinishNoFlags(pVCpu);
2218	}
2219
2220
2221	/**
2222	* Updates the EIP to point to the next instruction, but does not need to clear
2223	* EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.
2224	*
2225	* This is never from 64-bit code.
2226	*
2227	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2228	* @param cbInstr The number of bytes to add.
2229	*/
2230	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2231	{
2232	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2233	return iemRegFinishNoFlags(pVCpu);
2234	}
2235
2236
2237	/**
2238	* Updates the IP to point to the next instruction, but does not need to clear
2239	* EFLAGS.RF or CPUMCTX_INHIBIT_SHADOW nor check for debug flags.
2240	*
2241	* This is only ever used from 16-bit code on a pre-386 CPU.
2242	*
2243	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2244	* @param cbInstr The number of bytes to add.
2245	*/
2246	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2247	{
2248	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2249	return iemRegFinishNoFlags(pVCpu);
2250	}
2251
2252
2253	/**
2254	* Adds a 8-bit signed jump offset to RIP from 64-bit code.
2255	*
2256	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2257	* segment limit.
2258	*
2259	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2260	* @param cbInstr Instruction size.
2261	* @param offNextInstr The offset of the next instruction.
2262	* @param enmEffOpSize Effective operand size.
2263	*/
2264	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2265	IEMMODE enmEffOpSize) RT_NOEXCEPT
2266	{
2267	Assert(IEM_IS_64BIT_CODE(pVCpu));
2268	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2269
2270	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2271	if (enmEffOpSize == IEMMODE_16BIT)
2272	uNewRip &= UINT16_MAX;
2273
2274	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2275	pVCpu->cpum.GstCtx.rip = uNewRip;
2276	else
2277	return iemRaiseGeneralProtectionFault0(pVCpu);
2278
2279	#ifndef IEM_WITH_CODE_TLB
2280	iemOpcodeFlushLight(pVCpu, cbInstr);
2281	#endif
2282
2283	/*
2284	* Clear RF and finish the instruction (maybe raise #DB).
2285	*/
2286	return iemRegFinishClearingRF(pVCpu);
2287	}
2288
2289
2290	/**
2291	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2292	* code (never 64-bit).
2293	*
2294	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2295	* segment limit.
2296	*
2297	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2298	* @param cbInstr Instruction size.
2299	* @param offNextInstr The offset of the next instruction.
2300	* @param enmEffOpSize Effective operand size.
2301	*/
2302	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2303	IEMMODE enmEffOpSize) RT_NOEXCEPT
2304	{
2305	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2306	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2307
2308	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2309	if (enmEffOpSize == IEMMODE_16BIT)
2310	uNewEip &= UINT16_MAX;
2311	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2312	pVCpu->cpum.GstCtx.rip = uNewEip;
2313	else
2314	return iemRaiseGeneralProtectionFault0(pVCpu);
2315
2316	#ifndef IEM_WITH_CODE_TLB
2317	iemOpcodeFlushLight(pVCpu, cbInstr);
2318	#endif
2319
2320	/*
2321	* Clear RF and finish the instruction (maybe raise #DB).
2322	*/
2323	return iemRegFinishClearingRF(pVCpu);
2324	}
2325
2326
2327	/**
2328	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2329	*
2330	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2331	* segment limit.
2332	*
2333	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2334	* @param cbInstr Instruction size.
2335	* @param offNextInstr The offset of the next instruction.
2336	*/
2337	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2338	int8_t offNextInstr) RT_NOEXCEPT
2339	{
2340	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2341
2342	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2343	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2344	pVCpu->cpum.GstCtx.rip = uNewIp;
2345	else
2346	return iemRaiseGeneralProtectionFault0(pVCpu);
2347
2348	#ifndef IEM_WITH_CODE_TLB
2349	iemOpcodeFlushLight(pVCpu, cbInstr);
2350	#endif
2351
2352	/*
2353	* Clear RF and finish the instruction (maybe raise #DB).
2354	*/
2355	return iemRegFinishClearingRF(pVCpu);
2356	}
2357
2358
2359	/**
2360	* Adds a 8-bit signed jump offset to RIP from 64-bit code, no checking or
2361	* clearing of flags.
2362	*
2363	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2364	* segment limit.
2365	*
2366	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2367	* @param cbInstr Instruction size.
2368	* @param offNextInstr The offset of the next instruction.
2369	* @param enmEffOpSize Effective operand size.
2370	*/
2371	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2372	IEMMODE enmEffOpSize) RT_NOEXCEPT
2373	{
2374	Assert(IEM_IS_64BIT_CODE(pVCpu));
2375	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2376
2377	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2378	if (enmEffOpSize == IEMMODE_16BIT)
2379	uNewRip &= UINT16_MAX;
2380
2381	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2382	pVCpu->cpum.GstCtx.rip = uNewRip;
2383	else
2384	return iemRaiseGeneralProtectionFault0(pVCpu);
2385
2386	#ifndef IEM_WITH_CODE_TLB
2387	iemOpcodeFlushLight(pVCpu, cbInstr);
2388	#endif
2389	return iemRegFinishNoFlags(pVCpu);
2390	}
2391
2392
2393	/**
2394	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2395	* code (never 64-bit), no checking or clearing of flags.
2396	*
2397	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2398	* segment limit.
2399	*
2400	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2401	* @param cbInstr Instruction size.
2402	* @param offNextInstr The offset of the next instruction.
2403	* @param enmEffOpSize Effective operand size.
2404	*/
2405	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2406	IEMMODE enmEffOpSize) RT_NOEXCEPT
2407	{
2408	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2409	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2410
2411	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2412	if (enmEffOpSize == IEMMODE_16BIT)
2413	uNewEip &= UINT16_MAX;
2414	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2415	pVCpu->cpum.GstCtx.rip = uNewEip;
2416	else
2417	return iemRaiseGeneralProtectionFault0(pVCpu);
2418
2419	#ifndef IEM_WITH_CODE_TLB
2420	iemOpcodeFlushLight(pVCpu, cbInstr);
2421	#endif
2422	return iemRegFinishNoFlags(pVCpu);
2423	}
2424
2425
2426	/**
2427	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU, no checking or
2428	* clearing of flags.
2429	*
2430	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2431	* segment limit.
2432	*
2433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2434	* @param cbInstr Instruction size.
2435	* @param offNextInstr The offset of the next instruction.
2436	*/
2437	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2438	int8_t offNextInstr) RT_NOEXCEPT
2439	{
2440	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2441
2442	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2443	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2444	pVCpu->cpum.GstCtx.rip = uNewIp;
2445	else
2446	return iemRaiseGeneralProtectionFault0(pVCpu);
2447
2448	#ifndef IEM_WITH_CODE_TLB
2449	iemOpcodeFlushLight(pVCpu, cbInstr);
2450	#endif
2451	return iemRegFinishNoFlags(pVCpu);
2452	}
2453
2454
2455	/**
2456	* Adds a 16-bit signed jump offset to RIP from 64-bit code.
2457	*
2458	* @returns Strict VBox status code.
2459	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2460	* @param cbInstr Instruction size.
2461	* @param offNextInstr The offset of the next instruction.
2462	*/
2463	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2464	int16_t offNextInstr) RT_NOEXCEPT
2465	{
2466	Assert(IEM_IS_64BIT_CODE(pVCpu));
2467
2468	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2469
2470	#ifndef IEM_WITH_CODE_TLB
2471	iemOpcodeFlushLight(pVCpu, cbInstr);
2472	#endif
2473
2474	/*
2475	* Clear RF and finish the instruction (maybe raise #DB).
2476	*/
2477	return iemRegFinishClearingRF(pVCpu);
2478	}
2479
2480
2481	/**
2482	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2483	*
2484	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2485	* segment limit.
2486	*
2487	* @returns Strict VBox status code.
2488	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2489	* @param cbInstr Instruction size.
2490	* @param offNextInstr The offset of the next instruction.
2491	*
2492	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2493	* identical.
2494	*/
2495	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2496	int16_t offNextInstr) RT_NOEXCEPT
2497	{
2498	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2499
2500	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2501	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2502	pVCpu->cpum.GstCtx.rip = uNewIp;
2503	else
2504	return iemRaiseGeneralProtectionFault0(pVCpu);
2505
2506	#ifndef IEM_WITH_CODE_TLB
2507	iemOpcodeFlushLight(pVCpu, cbInstr);
2508	#endif
2509
2510	/*
2511	* Clear RF and finish the instruction (maybe raise #DB).
2512	*/
2513	return iemRegFinishClearingRF(pVCpu);
2514	}
2515
2516
2517	/**
2518	* Adds a 16-bit signed jump offset to RIP from 64-bit code, no checking or
2519	* clearing of flags.
2520	*
2521	* @returns Strict VBox status code.
2522	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2523	* @param cbInstr Instruction size.
2524	* @param offNextInstr The offset of the next instruction.
2525	*/
2526	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2527	int16_t offNextInstr) RT_NOEXCEPT
2528	{
2529	Assert(IEM_IS_64BIT_CODE(pVCpu));
2530
2531	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2532
2533	#ifndef IEM_WITH_CODE_TLB
2534	iemOpcodeFlushLight(pVCpu, cbInstr);
2535	#endif
2536	return iemRegFinishNoFlags(pVCpu);
2537	}
2538
2539
2540	/**
2541	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code,
2542	* no checking or clearing of flags.
2543	*
2544	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2545	* segment limit.
2546	*
2547	* @returns Strict VBox status code.
2548	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2549	* @param cbInstr Instruction size.
2550	* @param offNextInstr The offset of the next instruction.
2551	*
2552	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2553	* identical.
2554	*/
2555	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2556	int16_t offNextInstr) RT_NOEXCEPT
2557	{
2558	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2559
2560	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2561	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2562	pVCpu->cpum.GstCtx.rip = uNewIp;
2563	else
2564	return iemRaiseGeneralProtectionFault0(pVCpu);
2565
2566	#ifndef IEM_WITH_CODE_TLB
2567	iemOpcodeFlushLight(pVCpu, cbInstr);
2568	#endif
2569	return iemRegFinishNoFlags(pVCpu);
2570	}
2571
2572
2573	/**
2574	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2575	*
2576	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2577	* segment limit.
2578	*
2579	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2580	* only alternative for relative jumps in 64-bit code and that is already
2581	* handled in the decoder stage.
2582	*
2583	* @returns Strict VBox status code.
2584	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2585	* @param cbInstr Instruction size.
2586	* @param offNextInstr The offset of the next instruction.
2587	*/
2588	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2589	int32_t offNextInstr) RT_NOEXCEPT
2590	{
2591	Assert(IEM_IS_64BIT_CODE(pVCpu));
2592
2593	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2594	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2595	pVCpu->cpum.GstCtx.rip = uNewRip;
2596	else
2597	return iemRaiseGeneralProtectionFault0(pVCpu);
2598
2599	#ifndef IEM_WITH_CODE_TLB
2600	iemOpcodeFlushLight(pVCpu, cbInstr);
2601	#endif
2602
2603	/*
2604	* Clear RF and finish the instruction (maybe raise #DB).
2605	*/
2606	return iemRegFinishClearingRF(pVCpu);
2607	}
2608
2609
2610	/**
2611	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2612	*
2613	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2614	* segment limit.
2615	*
2616	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2617	* only alternative for relative jumps in 32-bit code and that is already
2618	* handled in the decoder stage.
2619	*
2620	* @returns Strict VBox status code.
2621	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2622	* @param cbInstr Instruction size.
2623	* @param offNextInstr The offset of the next instruction.
2624	*/
2625	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2626	int32_t offNextInstr) RT_NOEXCEPT
2627	{
2628	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2629	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2630
2631	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2632	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2633	pVCpu->cpum.GstCtx.rip = uNewEip;
2634	else
2635	return iemRaiseGeneralProtectionFault0(pVCpu);
2636
2637	#ifndef IEM_WITH_CODE_TLB
2638	iemOpcodeFlushLight(pVCpu, cbInstr);
2639	#endif
2640
2641	/*
2642	* Clear RF and finish the instruction (maybe raise #DB).
2643	*/
2644	return iemRegFinishClearingRF(pVCpu);
2645	}
2646
2647
2648	/**
2649	* Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or
2650	* clearing of flags.
2651	*
2652	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2653	* segment limit.
2654	*
2655	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2656	* only alternative for relative jumps in 64-bit code and that is already
2657	* handled in the decoder stage.
2658	*
2659	* @returns Strict VBox status code.
2660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2661	* @param cbInstr Instruction size.
2662	* @param offNextInstr The offset of the next instruction.
2663	*/
2664	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2665	int32_t offNextInstr) RT_NOEXCEPT
2666	{
2667	Assert(IEM_IS_64BIT_CODE(pVCpu));
2668
2669	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2670	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2671	pVCpu->cpum.GstCtx.rip = uNewRip;
2672	else
2673	return iemRaiseGeneralProtectionFault0(pVCpu);
2674
2675	#ifndef IEM_WITH_CODE_TLB
2676	iemOpcodeFlushLight(pVCpu, cbInstr);
2677	#endif
2678	return iemRegFinishNoFlags(pVCpu);
2679	}
2680
2681
2682	/**
2683	* Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or
2684	* clearing of flags.
2685	*
2686	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2687	* segment limit.
2688	*
2689	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2690	* only alternative for relative jumps in 32-bit code and that is already
2691	* handled in the decoder stage.
2692	*
2693	* @returns Strict VBox status code.
2694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2695	* @param cbInstr Instruction size.
2696	* @param offNextInstr The offset of the next instruction.
2697	*/
2698	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
2699	int32_t offNextInstr) RT_NOEXCEPT
2700	{
2701	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2702	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2703
2704	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2705	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2706	pVCpu->cpum.GstCtx.rip = uNewEip;
2707	else
2708	return iemRaiseGeneralProtectionFault0(pVCpu);
2709
2710	#ifndef IEM_WITH_CODE_TLB
2711	iemOpcodeFlushLight(pVCpu, cbInstr);
2712	#endif
2713	return iemRegFinishNoFlags(pVCpu);
2714	}
2715
2716
2717	/**
2718	* Extended version of iemFinishInstructionWithFlagsSet that goes with
2719	* iemRegAddToRipAndFinishingClearingRfEx.
2720	*
2721	* See iemFinishInstructionWithFlagsSet() for details.
2722	*/
2723	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2724	{
2725	/*
2726	* Raise a #DB.
2727	*/
2728	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2729	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2730	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
2731	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2732	/** @todo Do we set all pending \#DB events, or just one? */
2733	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2734	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2735	pVCpu->cpum.GstCtx.rflags.uBoth));
2736	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
2737	return iemRaiseDebugException(pVCpu);
2738	}
2739
2740
2741	/**
2742	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2743	* others potentially updating EFLAGS.TF.
2744	*
2745	* The single step event must be generated using the TF value at the start of
2746	* the instruction, not the new value set by it.
2747	*
2748	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2749	* @param cbInstr The number of bytes to add.
2750	* @param fEflOld The EFLAGS at the start of the instruction
2751	* execution.
2752	*/
2753	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2754	{
2755	iemRegAddToRip(pVCpu, cbInstr);
2756	if (!(fEflOld & X86_EFL_TF))
2757	return iemRegFinishClearingRF(pVCpu);
2758	return iemFinishInstructionWithTfSet(pVCpu);
2759	}
2760
2761
2762	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2763	/**
2764	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2765	*
2766	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2767	*/
2768	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2769	{
2770	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2771	}
2772	#endif
2773
2774
2775	#ifdef IEM_WITH_CODE_TLB
2776
2777	/**
2778	* Performs a near jump to the specified address, no checking or clearing of
2779	* flags
2780	*
2781	* May raise a \#GP(0) if the new IP outside the code segment limit.
2782	*
2783	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2784	* @param uNewIp The new IP value.
2785	*/
2786	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishNoFlags(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT
2787	{
2788	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
2789	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))
2790	pVCpu->cpum.GstCtx.rip = uNewIp;
2791	else
2792	return iemRaiseGeneralProtectionFault0(pVCpu);
2793	return iemRegFinishNoFlags(pVCpu);
2794	}
2795
2796
2797	/**
2798	* Performs a near jump to the specified address, no checking or clearing of
2799	* flags
2800	*
2801	* May raise a \#GP(0) if the new RIP is outside the code segment limit.
2802	*
2803	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2804	* @param uNewEip The new EIP value.
2805	*/
2806	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishNoFlags(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT
2807	{
2808	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2809	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2810	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2811	pVCpu->cpum.GstCtx.rip = uNewEip;
2812	else
2813	return iemRaiseGeneralProtectionFault0(pVCpu);
2814	return iemRegFinishNoFlags(pVCpu);
2815	}
2816
2817
2818	/**
2819	* Performs a near jump to the specified address, no checking or clearing of
2820	* flags.
2821	*
2822	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2823	* segment limit.
2824	*
2825	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2826	* @param uNewRip The new RIP value.
2827	*/
2828	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishNoFlags(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
2829	{
2830	Assert(IEM_IS_64BIT_CODE(pVCpu));
2831	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2832	pVCpu->cpum.GstCtx.rip = uNewRip;
2833	else
2834	return iemRaiseGeneralProtectionFault0(pVCpu);
2835	return iemRegFinishNoFlags(pVCpu);
2836	}
2837
2838	#endif /* IEM_WITH_CODE_TLB */
2839
2840	/**
2841	* Performs a near jump to the specified address.
2842	*
2843	* May raise a \#GP(0) if the new IP outside the code segment limit.
2844	*
2845	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2846	* @param uNewIp The new IP value.
2847	* @param cbInstr The instruction length, for flushing in the non-TLB case.
2848	*/
2849	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishClearingRF(PVMCPUCC pVCpu, uint16_t uNewIp, uint8_t cbInstr) RT_NOEXCEPT
2850	{
2851	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
2852	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))
2853	pVCpu->cpum.GstCtx.rip = uNewIp;
2854	else
2855	return iemRaiseGeneralProtectionFault0(pVCpu);
2856	#ifndef IEM_WITH_CODE_TLB
2857	iemOpcodeFlushLight(pVCpu, cbInstr);
2858	#else
2859	RT_NOREF_PV(cbInstr);
2860	#endif
2861	return iemRegFinishClearingRF(pVCpu);
2862	}
2863
2864
2865	/**
2866	* Performs a near jump to the specified address.
2867	*
2868	* May raise a \#GP(0) if the new RIP is outside the code segment limit.
2869	*
2870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2871	* @param uNewEip The new EIP value.
2872	* @param cbInstr The instruction length, for flushing in the non-TLB case.
2873	*/
2874	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishClearingRF(PVMCPUCC pVCpu, uint32_t uNewEip, uint8_t cbInstr) RT_NOEXCEPT
2875	{
2876	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2877	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2878	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2879	pVCpu->cpum.GstCtx.rip = uNewEip;
2880	else
2881	return iemRaiseGeneralProtectionFault0(pVCpu);
2882	#ifndef IEM_WITH_CODE_TLB
2883	iemOpcodeFlushLight(pVCpu, cbInstr);
2884	#else
2885	RT_NOREF_PV(cbInstr);
2886	#endif
2887	return iemRegFinishClearingRF(pVCpu);
2888	}
2889
2890
2891	/**
2892	* Performs a near jump to the specified address.
2893	*
2894	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2895	* segment limit.
2896	*
2897	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2898	* @param uNewRip The new RIP value.
2899	* @param cbInstr The instruction length, for flushing in the non-TLB case.
2900	*/
2901	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishClearingRF(PVMCPUCC pVCpu, uint64_t uNewRip, uint8_t cbInstr) RT_NOEXCEPT
2902	{
2903	Assert(IEM_IS_64BIT_CODE(pVCpu));
2904	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2905	pVCpu->cpum.GstCtx.rip = uNewRip;
2906	else
2907	return iemRaiseGeneralProtectionFault0(pVCpu);
2908	#ifndef IEM_WITH_CODE_TLB
2909	iemOpcodeFlushLight(pVCpu, cbInstr);
2910	#else
2911	RT_NOREF_PV(cbInstr);
2912	#endif
2913	return iemRegFinishClearingRF(pVCpu);
2914	}
2915
2916
2917
2918	/**
2919	* Adds to the stack pointer.
2920	*
2921	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2922	* @param cbToAdd The number of bytes to add (8-bit!).
2923	*/
2924	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2925	{
2926	if (IEM_IS_64BIT_CODE(pVCpu))
2927	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2928	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2929	pVCpu->cpum.GstCtx.esp += cbToAdd;
2930	else
2931	pVCpu->cpum.GstCtx.sp += cbToAdd;
2932	}
2933
2934
2935	/**
2936	* Subtracts from the stack pointer.
2937	*
2938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2939	* @param cbToSub The number of bytes to subtract (8-bit!).
2940	*/
2941	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2942	{
2943	if (IEM_IS_64BIT_CODE(pVCpu))
2944	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2945	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2946	pVCpu->cpum.GstCtx.esp -= cbToSub;
2947	else
2948	pVCpu->cpum.GstCtx.sp -= cbToSub;
2949	}
2950
2951
2952	/**
2953	* Adds to the temporary stack pointer.
2954	*
2955	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2956	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2957	* @param cbToAdd The number of bytes to add (16-bit).
2958	*/
2959	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2960	{
2961	if (IEM_IS_64BIT_CODE(pVCpu))
2962	pTmpRsp->u += cbToAdd;
2963	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2964	pTmpRsp->DWords.dw0 += cbToAdd;
2965	else
2966	pTmpRsp->Words.w0 += cbToAdd;
2967	}
2968
2969
2970	/**
2971	* Subtracts from the temporary stack pointer.
2972	*
2973	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2974	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2975	* @param cbToSub The number of bytes to subtract.
2976	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2977	* expecting that.
2978	*/
2979	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2980	{
2981	if (IEM_IS_64BIT_CODE(pVCpu))
2982	pTmpRsp->u -= cbToSub;
2983	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2984	pTmpRsp->DWords.dw0 -= cbToSub;
2985	else
2986	pTmpRsp->Words.w0 -= cbToSub;
2987	}
2988
2989
2990	/**
2991	* Calculates the effective stack address for a push of the specified size as
2992	* well as the new RSP value (upper bits may be masked).
2993	*
2994	* @returns Effective stack addressf for the push.
2995	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2996	* @param cbItem The size of the stack item to pop.
2997	* @param puNewRsp Where to return the new RSP value.
2998	*/
2999	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
3000	{
3001	RTUINT64U uTmpRsp;
3002	RTGCPTR GCPtrTop;
3003	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
3004
3005	if (IEM_IS_64BIT_CODE(pVCpu))
3006	GCPtrTop = uTmpRsp.u -= cbItem;
3007	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3008	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
3009	else
3010	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
3011	*puNewRsp = uTmpRsp.u;
3012	return GCPtrTop;
3013	}
3014
3015
3016	/**
3017	* Gets the current stack pointer and calculates the value after a pop of the
3018	* specified size.
3019	*
3020	* @returns Current stack pointer.
3021	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3022	* @param cbItem The size of the stack item to pop.
3023	* @param puNewRsp Where to return the new RSP value.
3024	*/
3025	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
3026	{
3027	RTUINT64U uTmpRsp;
3028	RTGCPTR GCPtrTop;
3029	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
3030
3031	if (IEM_IS_64BIT_CODE(pVCpu))
3032	{
3033	GCPtrTop = uTmpRsp.u;
3034	uTmpRsp.u += cbItem;
3035	}
3036	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3037	{
3038	GCPtrTop = uTmpRsp.DWords.dw0;
3039	uTmpRsp.DWords.dw0 += cbItem;
3040	}
3041	else
3042	{
3043	GCPtrTop = uTmpRsp.Words.w0;
3044	uTmpRsp.Words.w0 += cbItem;
3045	}
3046	*puNewRsp = uTmpRsp.u;
3047	return GCPtrTop;
3048	}
3049
3050
3051	/**
3052	* Calculates the effective stack address for a push of the specified size as
3053	* well as the new temporary RSP value (upper bits may be masked).
3054	*
3055	* @returns Effective stack addressf for the push.
3056	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3057	* @param pTmpRsp The temporary stack pointer. This is updated.
3058	* @param cbItem The size of the stack item to pop.
3059	*/
3060	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
3061	{
3062	RTGCPTR GCPtrTop;
3063
3064	if (IEM_IS_64BIT_CODE(pVCpu))
3065	GCPtrTop = pTmpRsp->u -= cbItem;
3066	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3067	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
3068	else
3069	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
3070	return GCPtrTop;
3071	}
3072
3073
3074	/**
3075	* Gets the effective stack address for a pop of the specified size and
3076	* calculates and updates the temporary RSP.
3077	*
3078	* @returns Current stack pointer.
3079	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3080	* @param pTmpRsp The temporary stack pointer. This is updated.
3081	* @param cbItem The size of the stack item to pop.
3082	*/
3083	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
3084	{
3085	RTGCPTR GCPtrTop;
3086	if (IEM_IS_64BIT_CODE(pVCpu))
3087	{
3088	GCPtrTop = pTmpRsp->u;
3089	pTmpRsp->u += cbItem;
3090	}
3091	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
3092	{
3093	GCPtrTop = pTmpRsp->DWords.dw0;
3094	pTmpRsp->DWords.dw0 += cbItem;
3095	}
3096	else
3097	{
3098	GCPtrTop = pTmpRsp->Words.w0;
3099	pTmpRsp->Words.w0 += cbItem;
3100	}
3101	return GCPtrTop;
3102	}
3103
3104	/** @} */
3105
3106
3107	/** @name FPU access and helpers.
3108	*
3109	* @{
3110	*/
3111
3112
3113	/**
3114	* Hook for preparing to use the host FPU.
3115	*
3116	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3117	*
3118	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3119	*/
3120	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
3121	{
3122	#ifdef IN_RING3
3123	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3124	#else
3125	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
3126	#endif
3127	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3128	}
3129
3130
3131	/**
3132	* Hook for preparing to use the host FPU for SSE.
3133	*
3134	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3135	*
3136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3137	*/
3138	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
3139	{
3140	iemFpuPrepareUsage(pVCpu);
3141	}
3142
3143
3144	/**
3145	* Hook for preparing to use the host FPU for AVX.
3146	*
3147	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3148	*
3149	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3150	*/
3151	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
3152	{
3153	iemFpuPrepareUsage(pVCpu);
3154	}
3155
3156
3157	/**
3158	* Hook for actualizing the guest FPU state before the interpreter reads it.
3159	*
3160	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3161	*
3162	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3163	*/
3164	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
3165	{
3166	#ifdef IN_RING3
3167	NOREF(pVCpu);
3168	#else
3169	CPUMRZFpuStateActualizeForRead(pVCpu);
3170	#endif
3171	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3172	}
3173
3174
3175	/**
3176	* Hook for actualizing the guest FPU state before the interpreter changes it.
3177	*
3178	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3179	*
3180	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3181	*/
3182	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
3183	{
3184	#ifdef IN_RING3
3185	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3186	#else
3187	CPUMRZFpuStateActualizeForChange(pVCpu);
3188	#endif
3189	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3190	}
3191
3192
3193	/**
3194	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
3195	* only.
3196	*
3197	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3198	*
3199	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3200	*/
3201	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
3202	{
3203	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
3204	NOREF(pVCpu);
3205	#else
3206	CPUMRZFpuStateActualizeSseForRead(pVCpu);
3207	#endif
3208	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3209	}
3210
3211
3212	/**
3213	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
3214	* read+write.
3215	*
3216	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3217	*
3218	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3219	*/
3220	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
3221	{
3222	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
3223	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3224	#else
3225	CPUMRZFpuStateActualizeForChange(pVCpu);
3226	#endif
3227	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3228
3229	/* Make sure any changes are loaded the next time around. */
3230	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
3231	}
3232
3233
3234	/**
3235	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
3236	* only.
3237	*
3238	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3239	*
3240	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3241	*/
3242	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
3243	{
3244	#ifdef IN_RING3
3245	NOREF(pVCpu);
3246	#else
3247	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
3248	#endif
3249	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3250	}
3251
3252
3253	/**
3254	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
3255	* read+write.
3256	*
3257	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
3258	*
3259	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3260	*/
3261	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
3262	{
3263	#ifdef IN_RING3
3264	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
3265	#else
3266	CPUMRZFpuStateActualizeForChange(pVCpu);
3267	#endif
3268	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
3269
3270	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
3271	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
3272	}
3273
3274
3275	/**
3276	* Stores a QNaN value into a FPU register.
3277	*
3278	* @param pReg Pointer to the register.
3279	*/
3280	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
3281	{
3282	pReg->au32[0] = UINT32_C(0x00000000);
3283	pReg->au32[1] = UINT32_C(0xc0000000);
3284	pReg->au16[4] = UINT16_C(0xffff);
3285	}
3286
3287
3288	/**
3289	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
3290	*
3291	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3292	* @param pFpuCtx The FPU context.
3293	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
3294	*/
3295	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
3296	{
3297	Assert(uFpuOpcode != UINT16_MAX);
3298	pFpuCtx->FOP = uFpuOpcode;
3299	/** @todo x87.CS and FPUIP needs to be kept seperately. */
3300	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
3301	{
3302	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
3303	* happens in real mode here based on the fnsave and fnstenv images. */
3304	pFpuCtx->CS = 0;
3305	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
3306	}
3307	else if (!IEM_IS_LONG_MODE(pVCpu))
3308	{
3309	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
3310	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
3311	}
3312	else
3313	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
3314	}
3315
3316
3317	/**
3318	* Marks the specified stack register as free (for FFREE).
3319	*
3320	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3321	* @param iStReg The register to free.
3322	*/
3323	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
3324	{
3325	Assert(iStReg < 8);
3326	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3327	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
3328	pFpuCtx->FTW &= ~RT_BIT(iReg);
3329	}
3330
3331
3332	/**
3333	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
3334	*
3335	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3336	*/
3337	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
3338	{
3339	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3340	uint16_t uFsw = pFpuCtx->FSW;
3341	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
3342	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3343	uFsw &= ~X86_FSW_TOP_MASK;
3344	uFsw \|= uTop;
3345	pFpuCtx->FSW = uFsw;
3346	}
3347
3348
3349	/**
3350	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
3351	*
3352	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3353	*/
3354	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
3355	{
3356	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3357	uint16_t uFsw = pFpuCtx->FSW;
3358	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
3359	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
3360	uFsw &= ~X86_FSW_TOP_MASK;
3361	uFsw \|= uTop;
3362	pFpuCtx->FSW = uFsw;
3363	}
3364
3365
3366
3367
3368	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
3369	{
3370	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3371	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
3372	if (pFpuCtx->FTW & RT_BIT(iReg))
3373	return VINF_SUCCESS;
3374	return VERR_NOT_FOUND;
3375	}
3376
3377
3378	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
3379	{
3380	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3381	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
3382	if (pFpuCtx->FTW & RT_BIT(iReg))
3383	{
3384	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
3385	return VINF_SUCCESS;
3386	}
3387	return VERR_NOT_FOUND;
3388	}
3389
3390
3391	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
3392	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
3393	{
3394	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3395	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3396	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
3397	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
3398	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
3399	{
3400	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
3401	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
3402	return VINF_SUCCESS;
3403	}
3404	return VERR_NOT_FOUND;
3405	}
3406
3407
3408	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
3409	{
3410	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
3411	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3412	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
3413	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
3414	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
3415	{
3416	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
3417	return VINF_SUCCESS;
3418	}
3419	return VERR_NOT_FOUND;
3420	}
3421
3422
3423	/**
3424	* Rotates the stack registers when setting new TOS.
3425	*
3426	* @param pFpuCtx The FPU context.
3427	* @param iNewTop New TOS value.
3428	* @remarks We only do this to speed up fxsave/fxrstor which
3429	* arrange the FP registers in stack order.
3430	* MUST be done before writing the new TOS (FSW).
3431	*/
3432	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
3433	{
3434	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3435	RTFLOAT80U ar80Temp[8];
3436
3437	if (iOldTop == iNewTop)
3438	return;
3439
3440	/* Unscrew the stack and get it into 'native' order. */
3441	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
3442	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
3443	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
3444	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
3445	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
3446	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
3447	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
3448	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3449
3450	/* Now rotate the stack to the new position. */
3451	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3452	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3453	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3454	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3455	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3456	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3457	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3458	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3459	}
3460
3461
3462	/**
3463	* Updates the FPU exception status after FCW is changed.
3464	*
3465	* @param pFpuCtx The FPU context.
3466	*/
3467	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3468	{
3469	uint16_t u16Fsw = pFpuCtx->FSW;
3470	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3471	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
3472	else
3473	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
3474	pFpuCtx->FSW = u16Fsw;
3475	}
3476
3477
3478	/**
3479	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3480	*
3481	* @returns The full FTW.
3482	* @param pFpuCtx The FPU context.
3483	*/
3484	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3485	{
3486	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3487	uint16_t u16Ftw = 0;
3488	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3489	for (unsigned iSt = 0; iSt < 8; iSt++)
3490	{
3491	unsigned const iReg = (iSt + iTop) & 7;
3492	if (!(u8Ftw & RT_BIT(iReg)))
3493	u16Ftw \|= 3 << (iReg * 2); /* empty */
3494	else
3495	{
3496	uint16_t uTag;
3497	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3498	if (pr80Reg->s.uExponent == 0x7fff)
3499	uTag = 2; /* Exponent is all 1's => Special. */
3500	else if (pr80Reg->s.uExponent == 0x0000)
3501	{
3502	if (pr80Reg->s.uMantissa == 0x0000)
3503	uTag = 1; /* All bits are zero => Zero. */
3504	else
3505	uTag = 2; /* Must be special. */
3506	}
3507	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3508	uTag = 0; /* Valid. */
3509	else
3510	uTag = 2; /* Must be special. */
3511
3512	u16Ftw \|= uTag << (iReg * 2);
3513	}
3514	}
3515
3516	return u16Ftw;
3517	}
3518
3519
3520	/**
3521	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3522	*
3523	* @returns The compressed FTW.
3524	* @param u16FullFtw The full FTW to convert.
3525	*/
3526	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3527	{
3528	uint8_t u8Ftw = 0;
3529	for (unsigned i = 0; i < 8; i++)
3530	{
3531	if ((u16FullFtw & 3) != 3 /empty/)
3532	u8Ftw \|= RT_BIT(i);
3533	u16FullFtw >>= 2;
3534	}
3535
3536	return u8Ftw;
3537	}
3538
3539	/** @} */
3540
3541
3542	/** @name Memory access.
3543	*
3544	* @{
3545	*/
3546
3547
3548	/**
3549	* Checks whether alignment checks are enabled or not.
3550	*
3551	* @returns true if enabled, false if not.
3552	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3553	*/
3554	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3555	{
3556	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3557	return IEM_GET_CPL(pVCpu) == 3
3558	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3559	}
3560
3561	/**
3562	* Checks if the given segment can be written to, raise the appropriate
3563	* exception if not.
3564	*
3565	* @returns VBox strict status code.
3566	*
3567	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3568	* @param pHid Pointer to the hidden register.
3569	* @param iSegReg The register number.
3570	* @param pu64BaseAddr Where to return the base address to use for the
3571	* segment. (In 64-bit code it may differ from the
3572	* base in the hidden segment.)
3573	*/
3574	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3575	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3576	{
3577	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3578
3579	if (IEM_IS_64BIT_CODE(pVCpu))
3580	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3581	else
3582	{
3583	if (!pHid->Attr.n.u1Present)
3584	{
3585	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3586	AssertRelease(uSel == 0);
3587	LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3588	return iemRaiseGeneralProtectionFault0(pVCpu);
3589	}
3590
3591	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3592	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3593	&& !IEM_IS_64BIT_CODE(pVCpu) )
3594	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3595	*pu64BaseAddr = pHid->u64Base;
3596	}
3597	return VINF_SUCCESS;
3598	}
3599
3600
3601	/**
3602	* Checks if the given segment can be read from, raise the appropriate
3603	* exception if not.
3604	*
3605	* @returns VBox strict status code.
3606	*
3607	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3608	* @param pHid Pointer to the hidden register.
3609	* @param iSegReg The register number.
3610	* @param pu64BaseAddr Where to return the base address to use for the
3611	* segment. (In 64-bit code it may differ from the
3612	* base in the hidden segment.)
3613	*/
3614	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3615	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3616	{
3617	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3618
3619	if (IEM_IS_64BIT_CODE(pVCpu))
3620	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3621	else
3622	{
3623	if (!pHid->Attr.n.u1Present)
3624	{
3625	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3626	AssertRelease(uSel == 0);
3627	LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3628	return iemRaiseGeneralProtectionFault0(pVCpu);
3629	}
3630
3631	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3632	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3633	*pu64BaseAddr = pHid->u64Base;
3634	}
3635	return VINF_SUCCESS;
3636	}
3637
3638
3639	/**
3640	* Maps a physical page.
3641	*
3642	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3643	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3644	* @param GCPhysMem The physical address.
3645	* @param fAccess The intended access.
3646	* @param ppvMem Where to return the mapping address.
3647	* @param pLock The PGM lock.
3648	*/
3649	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3650	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3651	{
3652	#ifdef IEM_LOG_MEMORY_WRITES
3653	if (fAccess & IEM_ACCESS_TYPE_WRITE)
3654	return VERR_PGM_PHYS_TLB_CATCH_ALL;
3655	#endif
3656
3657	/** @todo This API may require some improving later. A private deal with PGM
3658	* regarding locking and unlocking needs to be struct. A couple of TLBs
3659	* living in PGM, but with publicly accessible inlined access methods
3660	* could perhaps be an even better solution. */
3661	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3662	GCPhysMem,
3663	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3664	RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3665	ppvMem,
3666	pLock);
3667	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
3668	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3669
3670	return rc;
3671	}
3672
3673
3674	/**
3675	* Unmap a page previously mapped by iemMemPageMap.
3676	*
3677	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3678	* @param GCPhysMem The physical address.
3679	* @param fAccess The intended access.
3680	* @param pvMem What iemMemPageMap returned.
3681	* @param pLock The PGM lock.
3682	*/
3683	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3684	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3685	{
3686	NOREF(pVCpu);
3687	NOREF(GCPhysMem);
3688	NOREF(fAccess);
3689	NOREF(pvMem);
3690	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3691	}
3692
3693	#ifdef IEM_WITH_SETJMP
3694
3695	/** @todo slim this down */
3696	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3697	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3698	{
3699	Assert(cbMem >= 1);
3700	Assert(iSegReg < X86_SREG_COUNT);
3701
3702	/*
3703	* 64-bit mode is simpler.
3704	*/
3705	if (IEM_IS_64BIT_CODE(pVCpu))
3706	{
3707	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3708	{
3709	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3710	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3711	GCPtrMem += pSel->u64Base;
3712	}
3713
3714	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3715	return GCPtrMem;
3716	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3717	}
3718	/*
3719	* 16-bit and 32-bit segmentation.
3720	*/
3721	else if (iSegReg != UINT8_MAX)
3722	{
3723	/** @todo Does this apply to segments with 4G-1 limit? */
3724	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3725	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3726	{
3727	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3728	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3729	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3730	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
3731	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
3732	\| X86_SEL_TYPE_CODE))
3733	{
3734	case X86DESCATTR_P: /* readonly data, expand up */
3735	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
3736	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
3737	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
3738	/* expand up */
3739	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3740	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3741	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3742	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3743	break;
3744
3745	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3746	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
3747	/* expand down */
3748	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3749	&& ( pSel->Attr.n.u1DefBig
3750	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
3751	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3752	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3753	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3754	break;
3755
3756	default:
3757	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3758	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3759	break;
3760	}
3761	}
3762	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3763	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3764	}
3765	/*
3766	* 32-bit flat address.
3767	*/
3768	else
3769	return GCPtrMem;
3770	}
3771
3772
3773	/** @todo slim this down */
3774	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3775	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3776	{
3777	Assert(cbMem >= 1);
3778	Assert(iSegReg < X86_SREG_COUNT);
3779
3780	/*
3781	* 64-bit mode is simpler.
3782	*/
3783	if (IEM_IS_64BIT_CODE(pVCpu))
3784	{
3785	if (iSegReg >= X86_SREG_FS)
3786	{
3787	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3788	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3789	GCPtrMem += pSel->u64Base;
3790	}
3791
3792	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3793	return GCPtrMem;
3794	}
3795	/*
3796	* 16-bit and 32-bit segmentation.
3797	*/
3798	else
3799	{
3800	Assert(GCPtrMem <= UINT32_MAX);
3801	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3802	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3803	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3804	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
3805	if ( fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE) /* data, expand up */
3806	/** @todo explore exactly how the CS stuff works in real mode. See also
3807	* http://www.rcollins.org/Productivity/DescriptorCache.html and
3808	* http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */
3809	\|\| (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. / /* @todo testcase! */
3810	{
3811	/* expand up */
3812	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3813	if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit
3814	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3815	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3816	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3817	}
3818	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
3819	{
3820	/* expand down - the uppger boundary is defined by the B bit, not G. */
3821	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3822	if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit
3823	&& (pSel->Attr.n.u1DefBig \|\| GCPtrLast32 <= UINT32_C(0xffff))
3824	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3825	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3826	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3827	}
3828	else
3829	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3830	}
3831	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3832	}
3833
3834	#endif /* IEM_WITH_SETJMP */
3835
3836	/**
3837	* Fakes a long mode stack selector for SS = 0.
3838	*
3839	* @param pDescSs Where to return the fake stack descriptor.
3840	* @param uDpl The DPL we want.
3841	*/
3842	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3843	{
3844	pDescSs->Long.au64[0] = 0;
3845	pDescSs->Long.au64[1] = 0;
3846	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3847	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3848	pDescSs->Long.Gen.u2Dpl = uDpl;
3849	pDescSs->Long.Gen.u1Present = 1;
3850	pDescSs->Long.Gen.u1Long = 1;
3851	}
3852
3853
3854	/*
3855	* Unmap helpers.
3856	*/
3857
3858	#ifdef IEM_WITH_SETJMP
3859
3860	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3861	{
3862	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3863	if (RT_LIKELY(bMapInfo == 0))
3864	return;
3865	# endif
3866	iemMemCommitAndUnmapRwSafeJmp(pVCpu, bMapInfo);
3867	}
3868
3869
3870	DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3871	{
3872	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3873	if (RT_LIKELY(bMapInfo == 0))
3874	return;
3875	# endif
3876	iemMemCommitAndUnmapWoSafeJmp(pVCpu, bMapInfo);
3877	}
3878
3879
3880	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3881	{
3882	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3883	if (RT_LIKELY(bMapInfo == 0))
3884	return;
3885	# endif
3886	iemMemCommitAndUnmapRoSafeJmp(pVCpu, bMapInfo);
3887	}
3888
3889	DECLINLINE(void) iemMemRollbackAndUnmapWo(PVMCPUCC pVCpu, uint8_t bMapInfo) RT_NOEXCEPT
3890	{
3891	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3892	if (RT_LIKELY(bMapInfo == 0))
3893	return;
3894	# endif
3895	iemMemRollbackAndUnmapWoSafe(pVCpu, bMapInfo);
3896	}
3897
3898	#endif /* IEM_WITH_SETJMP */
3899
3900
3901	/*
3902	* Instantiate R/W inline templates.
3903	*/
3904
3905	/** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3906	* Used to check if an unaligned access is if within the page and won't
3907	* trigger an \#AC.
3908	*
3909	* This can be used to deal with misaligned accesses on platforms that are
3910	* senstive to such if desires.
3911	*/
3912	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3913	AssertCompile(((3U + 1U) << 16) == X86_CR0_AM);
3914	#if 1
3915	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \
3916	( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \
3917	&& !( (uint32_t)(a_pVCpu)->cpum.GstCtx.cr0 \
3918	& (a_pVCpu)->cpum.GstCtx.eflags.u \
3919	& ((IEM_GET_CPL((a_pVCpu)) + 1U) << 16) /* IEM_GET_CPL(a_pVCpu) == 3 ? X86_CR0_AM : 0 */ \
3920	& X86_CR0_AM) )
3921	#else
3922	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0
3923	#endif
3924
3925	#define TMPL_MEM_TYPE uint8_t
3926	#define TMPL_MEM_TYPE_ALIGN 0
3927	#define TMPL_MEM_TYPE_SIZE 1
3928	#define TMPL_MEM_FN_SUFF U8
3929	#define TMPL_MEM_FMT_TYPE "%#04x"
3930	#define TMPL_MEM_FMT_DESC "byte"
3931	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3932
3933	#define TMPL_MEM_WITH_STACK
3934
3935	#define TMPL_MEM_TYPE uint16_t
3936	#define TMPL_MEM_TYPE_ALIGN 1
3937	#define TMPL_MEM_TYPE_SIZE 2
3938	#define TMPL_MEM_FN_SUFF U16
3939	#define TMPL_MEM_FMT_TYPE "%#06x"
3940	#define TMPL_MEM_FMT_DESC "word"
3941	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3942
3943	#define TMPL_WITH_PUSH_SREG
3944	#define TMPL_MEM_TYPE uint32_t
3945	#define TMPL_MEM_TYPE_ALIGN 3
3946	#define TMPL_MEM_TYPE_SIZE 4
3947	#define TMPL_MEM_FN_SUFF U32
3948	#define TMPL_MEM_FMT_TYPE "%#010x"
3949	#define TMPL_MEM_FMT_DESC "dword"
3950	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3951	#undef TMPL_WITH_PUSH_SREG
3952
3953	#define TMPL_MEM_TYPE uint64_t
3954	#define TMPL_MEM_TYPE_ALIGN 7
3955	#define TMPL_MEM_TYPE_SIZE 8
3956	#define TMPL_MEM_FN_SUFF U64
3957	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3958	#define TMPL_MEM_FMT_DESC "qword"
3959	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3960
3961	#undef TMPL_MEM_WITH_STACK
3962
3963	#define TMPL_MEM_NO_STORE
3964	#define TMPL_MEM_NO_MAPPING
3965	#define TMPL_MEM_TYPE uint64_t
3966	#define TMPL_MEM_TYPE_ALIGN 15
3967	#define TMPL_MEM_TYPE_SIZE 8
3968	#define TMPL_MEM_FN_SUFF U64AlignedU128
3969	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3970	#define TMPL_MEM_FMT_DESC "qword"
3971	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3972
3973	#undef TMPL_MEM_NO_STORE
3974	#undef TMPL_MEM_NO_MAPPING
3975
3976	#define TMPL_MEM_TYPE RTFLOAT80U
3977	#define TMPL_MEM_TYPE_ALIGN 7
3978	#define TMPL_MEM_TYPE_SIZE 10
3979	#define TMPL_MEM_FN_SUFF R80
3980	#define TMPL_MEM_FMT_TYPE "%.10Rhxs"
3981	#define TMPL_MEM_FMT_DESC "tword"
3982	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3983
3984	#define TMPL_MEM_TYPE RTPBCD80U
3985	#define TMPL_MEM_TYPE_ALIGN 7 /** @todo RTPBCD80U alignment testcase */
3986	#define TMPL_MEM_TYPE_SIZE 10
3987	#define TMPL_MEM_FN_SUFF D80
3988	#define TMPL_MEM_FMT_TYPE "%.10Rhxs"
3989	#define TMPL_MEM_FMT_DESC "tword"
3990	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3991
3992	#define TMPL_MEM_TYPE RTUINT128U
3993	#define TMPL_MEM_TYPE_ALIGN 15
3994	#define TMPL_MEM_TYPE_SIZE 16
3995	#define TMPL_MEM_FN_SUFF U128
3996	#define TMPL_MEM_FMT_TYPE "%.16Rhxs"
3997	#define TMPL_MEM_FMT_DESC "dqword"
3998	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3999
4000	#undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
4001
4002	/** @} */
4003
4004
4005	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
4006
4007	/**
4008	* Gets CR0 fixed-0 bits in VMX operation.
4009	*
4010	* We do this rather than fetching what we report to the guest (in
4011	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
4012	* values regardless of whether unrestricted-guest feature is available on the CPU.
4013	*
4014	* @returns CR0 fixed-0 bits.
4015	* @param pVCpu The cross context virtual CPU structure.
4016	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
4017	* must be returned. When @c false, the CR0 fixed-0
4018	* bits for VMX root mode is returned.
4019	*
4020	*/
4021	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
4022	{
4023	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
4024
4025	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
4026	if ( fVmxNonRootMode
4027	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
4028	return VMX_V_CR0_FIXED0_UX;
4029	return VMX_V_CR0_FIXED0;
4030	}
4031
4032
4033	# ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */
4034	/**
4035	* Sets virtual-APIC write emulation as pending.
4036	*
4037	* @param pVCpu The cross context virtual CPU structure.
4038	* @param offApic The offset in the virtual-APIC page that was written.
4039	*/
4040	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
4041	{
4042	Assert(offApic < XAPIC_OFF_END + 4);
4043
4044	/*
4045	* Record the currently updated APIC offset, as we need this later for figuring
4046	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
4047	* as for supplying the exit qualification when causing an APIC-write VM-exit.
4048	*/
4049	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
4050
4051	/*
4052	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
4053	* virtualization or APIC-write emulation).
4054	*/
4055	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
4056	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
4057	}
4058	# endif /* XAPIC_OFF_END */
4059
4060	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
4061
4062	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 102734

Download in other formats: