IEMInline.h@ 100830

Last change on this file since 100830 was 100830, checked in by vboxsync, 16 months ago
VMM/IEM: Build fix for when IEM_WITH_SETJMP is undefined. bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 121.9 KB

Line
1	/* $Id: IEMInline.h 100830 2023-08-09 13:24:19Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	/* Deal with the cases that should be treated as VINF_SUCCESS first. */
49	if ( rcStrict == VINF_IEM_YIELD_PENDING_FF
50	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */
51	\|\| rcStrict == VINF_VMX_VMEXIT
52	#endif
53	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
54	\|\| rcStrict == VINF_SVM_VMEXIT
55	#endif
56	)
57	{
58	if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
59	rcStrict = VINF_SUCCESS;
60	else
61	{
62	pVCpu->iem.s.cRetPassUpStatus++;
63	rcStrict = pVCpu->iem.s.rcPassUp;
64	}
65	}
66	else if (RT_SUCCESS(rcStrict))
67	{
68	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
69	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
70	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
71	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
72	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
73	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
74	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
75	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
76	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
77	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
78	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
79	\|\| rcStrict == VINF_EM_RAW_TO_R3
80	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
81	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
82	/* raw-mode / virt handlers only: */
83	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
84	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
85	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
86	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
87	\|\| rcStrict == VINF_SELM_SYNC_GDT
88	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
89	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
90	/* nested hw.virt codes: */
91	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
92	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
93	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
94	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
95	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
96	if (rcPassUp == VINF_SUCCESS)
97	pVCpu->iem.s.cRetInfStatuses++;
98	else if ( rcPassUp < VINF_EM_FIRST
99	\|\| rcPassUp > VINF_EM_LAST
100	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
101	{
102	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
103	pVCpu->iem.s.cRetPassUpStatus++;
104	rcStrict = rcPassUp;
105	}
106	else
107	{
108	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
109	pVCpu->iem.s.cRetInfStatuses++;
110	}
111	}
112	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
113	pVCpu->iem.s.cRetAspectNotImplemented++;
114	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
115	pVCpu->iem.s.cRetInstrNotImplemented++;
116	else
117	pVCpu->iem.s.cRetErrStatuses++;
118	}
119	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
120	{
121	pVCpu->iem.s.cRetPassUpStatus++;
122	rcStrict = pVCpu->iem.s.rcPassUp;
123	}
124
125	/* Just clear it here as well. */
126	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
127
128	return rcStrict;
129	}
130
131
132	/**
133	* Sets the pass up status.
134	*
135	* @returns VINF_SUCCESS.
136	* @param pVCpu The cross context virtual CPU structure of the
137	* calling thread.
138	* @param rcPassUp The pass up status. Must be informational.
139	* VINF_SUCCESS is not allowed.
140	*/
141	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
142	{
143	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
144
145	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
146	if (rcOldPassUp == VINF_SUCCESS)
147	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148	/* If both are EM scheduling codes, use EM priority rules. */
149	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
150	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
151	{
152	if (rcPassUp < rcOldPassUp)
153	{
154	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
155	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
156	}
157	else
158	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
159	}
160	/* Override EM scheduling with specific status code. */
161	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
162	{
163	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
164	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
165	}
166	/* Don't override specific status code, first come first served. */
167	else
168	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
169	return VINF_SUCCESS;
170	}
171
172
173	/**
174	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
175	*
176	* Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
177	* reject expand down data segments and conforming code segments.
178	*
179	* ASSUMES that the CPU is in 32-bit mode.
180	*
181	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
182	* @param pVCpu The cross context virtual CPU structure of the
183	* calling thread.
184	* @sa iemCalc32BitFlatIndicatorEsDs
185	*/
186	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
187	{
188	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
189	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
190	\| pVCpu->cpum.GstCtx.cs.Attr.u
191	\| pVCpu->cpum.GstCtx.ss.Attr.u
192	\| pVCpu->cpum.GstCtx.ds.Attr.u)
193	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
194	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
195	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
196	\| (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
197	\| (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
198	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
199	== 0
200	&& ( pVCpu->cpum.GstCtx.es.u64Base
201	\| pVCpu->cpum.GstCtx.cs.u64Base
202	\| pVCpu->cpum.GstCtx.ss.u64Base
203	\| pVCpu->cpum.GstCtx.ds.u64Base)
204	== 0
205	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
206	? IEM_F_MODE_X86_32BIT_FLAT : 0;
207	}
208
209
210	/**
211	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
212	* flat already.
213	*
214	* This is used by sysenter.
215	*
216	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
217	* @param pVCpu The cross context virtual CPU structure of the
218	* calling thread.
219	* @sa iemCalc32BitFlatIndicator
220	*/
221	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
222	{
223	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
224	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
225	\| pVCpu->cpum.GstCtx.ds.Attr.u)
226	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
227	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
228	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
229	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
230	== 0
231	&& ( pVCpu->cpum.GstCtx.es.u64Base
232	\| pVCpu->cpum.GstCtx.ds.u64Base)
233	== 0
234	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
235	? IEM_F_MODE_X86_32BIT_FLAT : 0;
236	}
237
238
239	/**
240	* Calculates the IEM_F_MODE_XXX and CPL flags.
241	*
242	* @returns IEM_F_MODE_XXX
243	* @param pVCpu The cross context virtual CPU structure of the
244	* calling thread.
245	*/
246	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
247	{
248	/*
249	* We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
250	* here to try get this done as efficiently as possible.
251	*/
252	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
253
254	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
255	{
256	if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
257	{
258	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
259	uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
260	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
261	{
262	Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
263	fExec \|= IEM_F_MODE_X86_32BIT_PROT \| iemCalc32BitFlatIndicator(pVCpu);
264	}
265	else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
266	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
267	fExec \|= IEM_F_MODE_X86_64BIT;
268	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
269	fExec \|= IEM_F_MODE_X86_16BIT_PROT;
270	else
271	fExec \|= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
272	return fExec;
273	}
274	return IEM_F_MODE_X86_16BIT_PROT_V86 \| (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
275	}
276
277	/* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
278	if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
279	{
280	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
281	return IEM_F_MODE_X86_16BIT;
282	return IEM_F_MODE_X86_16BIT_PRE_386;
283	}
284
285	/* 32-bit unreal mode. */
286	return IEM_F_MODE_X86_32BIT \| iemCalc32BitFlatIndicator(pVCpu);
287	}
288
289
290	/**
291	* Calculates the AMD-V and VT-x related context flags.
292	*
293	* @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
294	* IEM_F_X86_CTX_VMX.
295	* @param pVCpu The cross context virtual CPU structure of the
296	* calling thread.
297	*/
298	DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
299	{
300	/*
301	* This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
302	* and CPUMIsGuestInNestedHwvirtMode to some extent.
303	*/
304	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
305
306	AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
307	uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
308	\| (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
309	if (RT_LIKELY(!fTmp))
310	return 0; /* likely */
311
312	if (fTmp & X86_CR4_VMXE)
313	{
314	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
315	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
316	return IEM_F_X86_CTX_VMX \| IEM_F_X86_CTX_IN_GUEST;
317	return IEM_F_X86_CTX_VMX;
318	}
319
320	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
321	if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
322	return IEM_F_X86_CTX_SVM \| IEM_F_X86_CTX_IN_GUEST;
323	return IEM_F_X86_CTX_SVM;
324	}
325
326
327	/**
328	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
329	*
330	* @returns IEM_F_BRK_PENDING_XXX or zero.
331	* @param pVCpu The cross context virtual CPU structure of the
332	* calling thread.
333	*/
334	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
335	{
336	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
337
338	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
339	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
340	return 0;
341	return iemCalcExecDbgFlagsSlow(pVCpu);
342	}
343
344	/**
345	* Calculates the the IEM_F_XXX flags.
346	*
347	* @returns IEM_F_XXX combination match the current CPU state.
348	* @param pVCpu The cross context virtual CPU structure of the
349	* calling thread.
350	*/
351	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
352	{
353	return iemCalcExecModeAndCplFlags(pVCpu)
354	\| iemCalcExecHwVirtFlags(pVCpu)
355	/* SMM is not yet implemented */
356	\| iemCalcExecDbgFlags(pVCpu)
357	;
358	}
359
360
361	/**
362	* Re-calculates the MODE and CPL parts of IEMCPU::fExec.
363	*
364	* @param pVCpu The cross context virtual CPU structure of the
365	* calling thread.
366	*/
367	DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
368	{
369	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
370	\| iemCalcExecModeAndCplFlags(pVCpu);
371	}
372
373
374	/**
375	* Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
376	*
377	* @param pVCpu The cross context virtual CPU structure of the
378	* calling thread.
379	*/
380	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
381	{
382	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
383	\| iemCalcExecDbgFlags(pVCpu);
384	}
385
386
387	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
388
389	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
390	/**
391	* Initializes the execution state.
392	*
393	* @param pVCpu The cross context virtual CPU structure of the
394	* calling thread.
395	* @param fExecOpts Optional execution flags:
396	* - IEM_F_BYPASS_HANDLERS
397	* - IEM_F_X86_DISREGARD_LOCK
398	*
399	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
400	* side-effects in strict builds.
401	*/
402	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
403	{
404	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
405	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
406	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
407	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
408	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
409	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
410	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
411	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
412	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
413	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
414
415	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
416	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
417	pVCpu->iem.s.cActiveMappings = 0;
418	pVCpu->iem.s.iNextMapping = 0;
419
420	# ifdef VBOX_STRICT
421	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
422	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
423	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
424	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
425	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
426	pVCpu->iem.s.uRexReg = 127;
427	pVCpu->iem.s.uRexB = 127;
428	pVCpu->iem.s.offModRm = 127;
429	pVCpu->iem.s.uRexIndex = 127;
430	pVCpu->iem.s.iEffSeg = 127;
431	pVCpu->iem.s.idxPrefix = 127;
432	pVCpu->iem.s.uVex3rdReg = 127;
433	pVCpu->iem.s.uVexLength = 127;
434	pVCpu->iem.s.fEvexStuff = 127;
435	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
436	# ifdef IEM_WITH_CODE_TLB
437	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
438	pVCpu->iem.s.pbInstrBuf = NULL;
439	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
440	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
441	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
442	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
443	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
444	pVCpu->iem.s.offOpcode = 127;
445	# endif
446	# else
447	pVCpu->iem.s.offOpcode = 127;
448	pVCpu->iem.s.cbOpcode = 127;
449	# endif
450	# endif /* VBOX_STRICT */
451	}
452	# endif /* VBOX_INCLUDED_vmm_dbgf_h */
453
454
455	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
456	/**
457	* Performs a minimal reinitialization of the execution state.
458	*
459	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
460	* 'world-switch' types operations on the CPU. Currently only nested
461	* hardware-virtualization uses it.
462	*
463	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
464	* @param cbInstr The instruction length (for flushing).
465	*/
466	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
467	{
468	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
469	iemOpcodeFlushHeavy(pVCpu, cbInstr);
470	}
471	# endif
472
473
474	/**
475	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
476	*
477	* @param pVCpu The cross context virtual CPU structure of the
478	* calling thread.
479	*/
480	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
481	{
482	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
483	# ifdef VBOX_STRICT
484	# ifdef IEM_WITH_CODE_TLB
485	NOREF(pVCpu);
486	# else
487	pVCpu->iem.s.cbOpcode = 0;
488	# endif
489	# else
490	NOREF(pVCpu);
491	# endif
492	}
493
494
495	/**
496	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
497	*
498	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
499	*
500	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
501	* @param pVCpu The cross context virtual CPU structure of the calling thread.
502	* @param rcStrict The status code to fiddle.
503	*/
504	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
505	{
506	iemUninitExec(pVCpu);
507	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
508	}
509
510
511	/**
512	* Macro used by the IEMExec* method to check the given instruction length.
513	*
514	* Will return on failure!
515	*
516	* @param a_cbInstr The given instruction length.
517	* @param a_cbMin The minimum length.
518	*/
519	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
520	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
521	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
522
523
524	# ifndef IEM_WITH_SETJMP
525
526	/**
527	* Fetches the first opcode byte.
528	*
529	* @returns Strict VBox status code.
530	* @param pVCpu The cross context virtual CPU structure of the
531	* calling thread.
532	* @param pu8 Where to return the opcode byte.
533	*/
534	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
535	{
536	/*
537	* Check for hardware instruction breakpoints.
538	*/
539	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
540	{ /* likely */ }
541	else
542	{
543	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
544	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
545	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
546	{ /* likely */ }
547	else
548	{
549	pu8 = 0xff; / shut up gcc. sigh */
550	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
551	return iemRaiseDebugException(pVCpu);
552	return rcStrict;
553	}
554	}
555
556	/*
557	* Fetch the first opcode byte.
558	*/
559	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
560	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
561	{
562	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
563	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
564	return VINF_SUCCESS;
565	}
566	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
567	}
568
569	# else /* IEM_WITH_SETJMP */
570
571	/**
572	* Fetches the first opcode byte, longjmp on error.
573	*
574	* @returns The opcode byte.
575	* @param pVCpu The cross context virtual CPU structure of the calling thread.
576	*/
577	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
578	{
579	/*
580	* Check for hardware instruction breakpoints.
581	*/
582	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
583	{ /* likely */ }
584	else
585	{
586	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
587	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
588	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
589	{ /* likely */ }
590	else
591	{
592	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
593	rcStrict = iemRaiseDebugException(pVCpu);
594	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
595	}
596	}
597
598	/*
599	* Fetch the first opcode byte.
600	*/
601	# ifdef IEM_WITH_CODE_TLB
602	uint8_t bRet;
603	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
604	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
605	if (RT_LIKELY( pbBuf != NULL
606	&& offBuf < pVCpu->iem.s.cbInstrBuf))
607	{
608	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
609	bRet = pbBuf[offBuf];
610	}
611	else
612	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
613	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
614	Assert(pVCpu->iem.s.offOpcode == 0);
615	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
616	# endif
617	return bRet;
618
619	# else /* !IEM_WITH_CODE_TLB */
620	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
621	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
622	{
623	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
624	return pVCpu->iem.s.abOpcode[offOpcode];
625	}
626	return iemOpcodeGetNextU8SlowJmp(pVCpu);
627	# endif
628	}
629
630	# endif /* IEM_WITH_SETJMP */
631
632	/**
633	* Fetches the first opcode byte, returns/throws automatically on failure.
634	*
635	* @param a_pu8 Where to return the opcode byte.
636	* @remark Implicitly references pVCpu.
637	*/
638	# ifndef IEM_WITH_SETJMP
639	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
640	do \
641	{ \
642	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
643	if (rcStrict2 == VINF_SUCCESS) \
644	{ /* likely */ } \
645	else \
646	return rcStrict2; \
647	} while (0)
648	# else
649	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
650	# endif /* IEM_WITH_SETJMP */
651
652
653	# ifndef IEM_WITH_SETJMP
654
655	/**
656	* Fetches the next opcode byte.
657	*
658	* @returns Strict VBox status code.
659	* @param pVCpu The cross context virtual CPU structure of the
660	* calling thread.
661	* @param pu8 Where to return the opcode byte.
662	*/
663	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
664	{
665	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
666	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
667	{
668	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
669	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
670	return VINF_SUCCESS;
671	}
672	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
673	}
674
675	# else /* IEM_WITH_SETJMP */
676
677	/**
678	* Fetches the next opcode byte, longjmp on error.
679	*
680	* @returns The opcode byte.
681	* @param pVCpu The cross context virtual CPU structure of the calling thread.
682	*/
683	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
684	{
685	# ifdef IEM_WITH_CODE_TLB
686	uint8_t bRet;
687	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
688	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
689	if (RT_LIKELY( pbBuf != NULL
690	&& offBuf < pVCpu->iem.s.cbInstrBuf))
691	{
692	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
693	bRet = pbBuf[offBuf];
694	}
695	else
696	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
697	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
698	Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
699	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
700	# endif
701	return bRet;
702
703	# else /* !IEM_WITH_CODE_TLB */
704	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
705	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
706	{
707	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
708	return pVCpu->iem.s.abOpcode[offOpcode];
709	}
710	return iemOpcodeGetNextU8SlowJmp(pVCpu);
711	# endif
712	}
713
714	# endif /* IEM_WITH_SETJMP */
715
716	/**
717	* Fetches the next opcode byte, returns automatically on failure.
718	*
719	* @param a_pu8 Where to return the opcode byte.
720	* @remark Implicitly references pVCpu.
721	*/
722	# ifndef IEM_WITH_SETJMP
723	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
724	do \
725	{ \
726	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
727	if (rcStrict2 == VINF_SUCCESS) \
728	{ /* likely */ } \
729	else \
730	return rcStrict2; \
731	} while (0)
732	# else
733	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
734	# endif /* IEM_WITH_SETJMP */
735
736
737	# ifndef IEM_WITH_SETJMP
738	/**
739	* Fetches the next signed byte from the opcode stream.
740	*
741	* @returns Strict VBox status code.
742	* @param pVCpu The cross context virtual CPU structure of the calling thread.
743	* @param pi8 Where to return the signed byte.
744	*/
745	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
746	{
747	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
748	}
749	# endif /* !IEM_WITH_SETJMP */
750
751
752	/**
753	* Fetches the next signed byte from the opcode stream, returning automatically
754	* on failure.
755	*
756	* @param a_pi8 Where to return the signed byte.
757	* @remark Implicitly references pVCpu.
758	*/
759	# ifndef IEM_WITH_SETJMP
760	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
761	do \
762	{ \
763	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
764	if (rcStrict2 != VINF_SUCCESS) \
765	return rcStrict2; \
766	} while (0)
767	# else /* IEM_WITH_SETJMP */
768	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
769
770	# endif /* IEM_WITH_SETJMP */
771
772
773	# ifndef IEM_WITH_SETJMP
774	/**
775	* Fetches the next signed byte from the opcode stream, extending it to
776	* unsigned 16-bit.
777	*
778	* @returns Strict VBox status code.
779	* @param pVCpu The cross context virtual CPU structure of the calling thread.
780	* @param pu16 Where to return the unsigned word.
781	*/
782	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
783	{
784	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
785	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
786	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
787
788	*pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
789	pVCpu->iem.s.offOpcode = offOpcode + 1;
790	return VINF_SUCCESS;
791	}
792	# endif /* !IEM_WITH_SETJMP */
793
794	/**
795	* Fetches the next signed byte from the opcode stream and sign-extending it to
796	* a word, returning automatically on failure.
797	*
798	* @param a_pu16 Where to return the word.
799	* @remark Implicitly references pVCpu.
800	*/
801	# ifndef IEM_WITH_SETJMP
802	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
803	do \
804	{ \
805	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
806	if (rcStrict2 != VINF_SUCCESS) \
807	return rcStrict2; \
808	} while (0)
809	# else
810	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
811	# endif
812
813	# ifndef IEM_WITH_SETJMP
814	/**
815	* Fetches the next signed byte from the opcode stream, extending it to
816	* unsigned 32-bit.
817	*
818	* @returns Strict VBox status code.
819	* @param pVCpu The cross context virtual CPU structure of the calling thread.
820	* @param pu32 Where to return the unsigned dword.
821	*/
822	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
823	{
824	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
825	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
826	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
827
828	*pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
829	pVCpu->iem.s.offOpcode = offOpcode + 1;
830	return VINF_SUCCESS;
831	}
832	# endif /* !IEM_WITH_SETJMP */
833
834	/**
835	* Fetches the next signed byte from the opcode stream and sign-extending it to
836	* a word, returning automatically on failure.
837	*
838	* @param a_pu32 Where to return the word.
839	* @remark Implicitly references pVCpu.
840	*/
841	# ifndef IEM_WITH_SETJMP
842	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
843	do \
844	{ \
845	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
846	if (rcStrict2 != VINF_SUCCESS) \
847	return rcStrict2; \
848	} while (0)
849	# else
850	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
851	# endif
852
853
854	# ifndef IEM_WITH_SETJMP
855	/**
856	* Fetches the next signed byte from the opcode stream, extending it to
857	* unsigned 64-bit.
858	*
859	* @returns Strict VBox status code.
860	* @param pVCpu The cross context virtual CPU structure of the calling thread.
861	* @param pu64 Where to return the unsigned qword.
862	*/
863	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
864	{
865	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
866	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
867	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
868
869	*pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
870	pVCpu->iem.s.offOpcode = offOpcode + 1;
871	return VINF_SUCCESS;
872	}
873	# endif /* !IEM_WITH_SETJMP */
874
875	/**
876	* Fetches the next signed byte from the opcode stream and sign-extending it to
877	* a word, returning automatically on failure.
878	*
879	* @param a_pu64 Where to return the word.
880	* @remark Implicitly references pVCpu.
881	*/
882	# ifndef IEM_WITH_SETJMP
883	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
884	do \
885	{ \
886	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
887	if (rcStrict2 != VINF_SUCCESS) \
888	return rcStrict2; \
889	} while (0)
890	# else
891	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
892	# endif
893
894
895	# ifndef IEM_WITH_SETJMP
896
897	/**
898	* Fetches the next opcode word.
899	*
900	* @returns Strict VBox status code.
901	* @param pVCpu The cross context virtual CPU structure of the calling thread.
902	* @param pu16 Where to return the opcode word.
903	*/
904	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
905	{
906	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
907	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
908	{
909	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
910	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
911	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
912	# else
913	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
914	# endif
915	return VINF_SUCCESS;
916	}
917	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
918	}
919
920	# else /* IEM_WITH_SETJMP */
921
922	/**
923	* Fetches the next opcode word, longjmp on error.
924	*
925	* @returns The opcode word.
926	* @param pVCpu The cross context virtual CPU structure of the calling thread.
927	*/
928	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
929	{
930	# ifdef IEM_WITH_CODE_TLB
931	uint16_t u16Ret;
932	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
933	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
934	if (RT_LIKELY( pbBuf != NULL
935	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
936	{
937	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
938	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
939	u16Ret = (uint16_t const )&pbBuf[offBuf];
940	# else
941	u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
942	# endif
943	}
944	else
945	u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
946
947	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
948	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
949	Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
950	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
951	(uint16_t )&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
952	# else
953	pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
954	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
955	# endif
956	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
957	# endif
958
959	return u16Ret;
960
961	# else /* !IEM_WITH_CODE_TLB */
962	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
963	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
964	{
965	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
966	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
967	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
968	# else
969	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
970	# endif
971	}
972	return iemOpcodeGetNextU16SlowJmp(pVCpu);
973	# endif /* !IEM_WITH_CODE_TLB */
974	}
975
976	# endif /* IEM_WITH_SETJMP */
977
978	/**
979	* Fetches the next opcode word, returns automatically on failure.
980	*
981	* @param a_pu16 Where to return the opcode word.
982	* @remark Implicitly references pVCpu.
983	*/
984	# ifndef IEM_WITH_SETJMP
985	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
986	do \
987	{ \
988	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
989	if (rcStrict2 != VINF_SUCCESS) \
990	return rcStrict2; \
991	} while (0)
992	# else
993	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
994	# endif
995
996	# ifndef IEM_WITH_SETJMP
997	/**
998	* Fetches the next opcode word, zero extending it to a double word.
999	*
1000	* @returns Strict VBox status code.
1001	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1002	* @param pu32 Where to return the opcode double word.
1003	*/
1004	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1005	{
1006	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1007	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1008	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1009
1010	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1011	pVCpu->iem.s.offOpcode = offOpcode + 2;
1012	return VINF_SUCCESS;
1013	}
1014	# endif /* !IEM_WITH_SETJMP */
1015
1016	/**
1017	* Fetches the next opcode word and zero extends it to a double word, returns
1018	* automatically on failure.
1019	*
1020	* @param a_pu32 Where to return the opcode double word.
1021	* @remark Implicitly references pVCpu.
1022	*/
1023	# ifndef IEM_WITH_SETJMP
1024	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1025	do \
1026	{ \
1027	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1028	if (rcStrict2 != VINF_SUCCESS) \
1029	return rcStrict2; \
1030	} while (0)
1031	# else
1032	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1033	# endif
1034
1035	# ifndef IEM_WITH_SETJMP
1036	/**
1037	* Fetches the next opcode word, zero extending it to a quad word.
1038	*
1039	* @returns Strict VBox status code.
1040	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1041	* @param pu64 Where to return the opcode quad word.
1042	*/
1043	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1044	{
1045	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1046	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1047	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1048
1049	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1050	pVCpu->iem.s.offOpcode = offOpcode + 2;
1051	return VINF_SUCCESS;
1052	}
1053	# endif /* !IEM_WITH_SETJMP */
1054
1055	/**
1056	* Fetches the next opcode word and zero extends it to a quad word, returns
1057	* automatically on failure.
1058	*
1059	* @param a_pu64 Where to return the opcode quad word.
1060	* @remark Implicitly references pVCpu.
1061	*/
1062	# ifndef IEM_WITH_SETJMP
1063	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1064	do \
1065	{ \
1066	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1067	if (rcStrict2 != VINF_SUCCESS) \
1068	return rcStrict2; \
1069	} while (0)
1070	# else
1071	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1072	# endif
1073
1074
1075	# ifndef IEM_WITH_SETJMP
1076	/**
1077	* Fetches the next signed word from the opcode stream.
1078	*
1079	* @returns Strict VBox status code.
1080	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1081	* @param pi16 Where to return the signed word.
1082	*/
1083	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1084	{
1085	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1086	}
1087	# endif /* !IEM_WITH_SETJMP */
1088
1089
1090	/**
1091	* Fetches the next signed word from the opcode stream, returning automatically
1092	* on failure.
1093	*
1094	* @param a_pi16 Where to return the signed word.
1095	* @remark Implicitly references pVCpu.
1096	*/
1097	# ifndef IEM_WITH_SETJMP
1098	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1099	do \
1100	{ \
1101	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1102	if (rcStrict2 != VINF_SUCCESS) \
1103	return rcStrict2; \
1104	} while (0)
1105	# else
1106	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1107	# endif
1108
1109	# ifndef IEM_WITH_SETJMP
1110
1111	/**
1112	* Fetches the next opcode dword.
1113	*
1114	* @returns Strict VBox status code.
1115	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1116	* @param pu32 Where to return the opcode double word.
1117	*/
1118	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1119	{
1120	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1121	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1122	{
1123	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1124	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1125	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1126	# else
1127	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1128	pVCpu->iem.s.abOpcode[offOpcode + 1],
1129	pVCpu->iem.s.abOpcode[offOpcode + 2],
1130	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1131	# endif
1132	return VINF_SUCCESS;
1133	}
1134	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1135	}
1136
1137	# else /* IEM_WITH_SETJMP */
1138
1139	/**
1140	* Fetches the next opcode dword, longjmp on error.
1141	*
1142	* @returns The opcode dword.
1143	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1144	*/
1145	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1146	{
1147	# ifdef IEM_WITH_CODE_TLB
1148	uint32_t u32Ret;
1149	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1150	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1151	if (RT_LIKELY( pbBuf != NULL
1152	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1153	{
1154	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1155	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1156	u32Ret = (uint32_t const )&pbBuf[offBuf];
1157	# else
1158	u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1159	pbBuf[offBuf + 1],
1160	pbBuf[offBuf + 2],
1161	pbBuf[offBuf + 3]);
1162	# endif
1163	}
1164	else
1165	u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1166
1167	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1168	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1169	Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1170	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1171	(uint32_t )&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1172	# else
1173	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1174	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1175	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1176	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1177	# endif
1178	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1179	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1180
1181	return u32Ret;
1182
1183	# else /* !IEM_WITH_CODE_TLB */
1184	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1185	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1186	{
1187	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1188	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1189	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1190	# else
1191	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1192	pVCpu->iem.s.abOpcode[offOpcode + 1],
1193	pVCpu->iem.s.abOpcode[offOpcode + 2],
1194	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1195	# endif
1196	}
1197	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1198	# endif
1199	}
1200
1201	# endif /* IEM_WITH_SETJMP */
1202
1203	/**
1204	* Fetches the next opcode dword, returns automatically on failure.
1205	*
1206	* @param a_pu32 Where to return the opcode dword.
1207	* @remark Implicitly references pVCpu.
1208	*/
1209	# ifndef IEM_WITH_SETJMP
1210	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1211	do \
1212	{ \
1213	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1214	if (rcStrict2 != VINF_SUCCESS) \
1215	return rcStrict2; \
1216	} while (0)
1217	# else
1218	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1219	# endif
1220
1221	# ifndef IEM_WITH_SETJMP
1222	/**
1223	* Fetches the next opcode dword, zero extending it to a quad word.
1224	*
1225	* @returns Strict VBox status code.
1226	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1227	* @param pu64 Where to return the opcode quad word.
1228	*/
1229	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1230	{
1231	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1232	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1233	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1234
1235	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1236	pVCpu->iem.s.abOpcode[offOpcode + 1],
1237	pVCpu->iem.s.abOpcode[offOpcode + 2],
1238	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1239	pVCpu->iem.s.offOpcode = offOpcode + 4;
1240	return VINF_SUCCESS;
1241	}
1242	# endif /* !IEM_WITH_SETJMP */
1243
1244	/**
1245	* Fetches the next opcode dword and zero extends it to a quad word, returns
1246	* automatically on failure.
1247	*
1248	* @param a_pu64 Where to return the opcode quad word.
1249	* @remark Implicitly references pVCpu.
1250	*/
1251	# ifndef IEM_WITH_SETJMP
1252	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1253	do \
1254	{ \
1255	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1256	if (rcStrict2 != VINF_SUCCESS) \
1257	return rcStrict2; \
1258	} while (0)
1259	# else
1260	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1261	# endif
1262
1263
1264	# ifndef IEM_WITH_SETJMP
1265	/**
1266	* Fetches the next signed double word from the opcode stream.
1267	*
1268	* @returns Strict VBox status code.
1269	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1270	* @param pi32 Where to return the signed double word.
1271	*/
1272	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1273	{
1274	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1275	}
1276	# endif
1277
1278	/**
1279	* Fetches the next signed double word from the opcode stream, returning
1280	* automatically on failure.
1281	*
1282	* @param a_pi32 Where to return the signed double word.
1283	* @remark Implicitly references pVCpu.
1284	*/
1285	# ifndef IEM_WITH_SETJMP
1286	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1287	do \
1288	{ \
1289	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1290	if (rcStrict2 != VINF_SUCCESS) \
1291	return rcStrict2; \
1292	} while (0)
1293	# else
1294	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1295	# endif
1296
1297	# ifndef IEM_WITH_SETJMP
1298	/**
1299	* Fetches the next opcode dword, sign extending it into a quad word.
1300	*
1301	* @returns Strict VBox status code.
1302	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1303	* @param pu64 Where to return the opcode quad word.
1304	*/
1305	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1306	{
1307	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1308	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1309	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1310
1311	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1312	pVCpu->iem.s.abOpcode[offOpcode + 1],
1313	pVCpu->iem.s.abOpcode[offOpcode + 2],
1314	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1315	*pu64 = (uint64_t)(int64_t)i32;
1316	pVCpu->iem.s.offOpcode = offOpcode + 4;
1317	return VINF_SUCCESS;
1318	}
1319	# endif /* !IEM_WITH_SETJMP */
1320
1321	/**
1322	* Fetches the next opcode double word and sign extends it to a quad word,
1323	* returns automatically on failure.
1324	*
1325	* @param a_pu64 Where to return the opcode quad word.
1326	* @remark Implicitly references pVCpu.
1327	*/
1328	# ifndef IEM_WITH_SETJMP
1329	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1330	do \
1331	{ \
1332	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1333	if (rcStrict2 != VINF_SUCCESS) \
1334	return rcStrict2; \
1335	} while (0)
1336	# else
1337	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1338	# endif
1339
1340	# ifndef IEM_WITH_SETJMP
1341
1342	/**
1343	* Fetches the next opcode qword.
1344	*
1345	* @returns Strict VBox status code.
1346	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1347	* @param pu64 Where to return the opcode qword.
1348	*/
1349	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1350	{
1351	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1352	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1353	{
1354	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1355	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1356	# else
1357	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1358	pVCpu->iem.s.abOpcode[offOpcode + 1],
1359	pVCpu->iem.s.abOpcode[offOpcode + 2],
1360	pVCpu->iem.s.abOpcode[offOpcode + 3],
1361	pVCpu->iem.s.abOpcode[offOpcode + 4],
1362	pVCpu->iem.s.abOpcode[offOpcode + 5],
1363	pVCpu->iem.s.abOpcode[offOpcode + 6],
1364	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1365	# endif
1366	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1367	return VINF_SUCCESS;
1368	}
1369	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1370	}
1371
1372	# else /* IEM_WITH_SETJMP */
1373
1374	/**
1375	* Fetches the next opcode qword, longjmp on error.
1376	*
1377	* @returns The opcode qword.
1378	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1379	*/
1380	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1381	{
1382	# ifdef IEM_WITH_CODE_TLB
1383	uint64_t u64Ret;
1384	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1385	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1386	if (RT_LIKELY( pbBuf != NULL
1387	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1388	{
1389	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1390	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1391	u64Ret = (uint64_t const )&pbBuf[offBuf];
1392	# else
1393	u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1394	pbBuf[offBuf + 1],
1395	pbBuf[offBuf + 2],
1396	pbBuf[offBuf + 3],
1397	pbBuf[offBuf + 4],
1398	pbBuf[offBuf + 5],
1399	pbBuf[offBuf + 6],
1400	pbBuf[offBuf + 7]);
1401	# endif
1402	}
1403	else
1404	u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1405
1406	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1407	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1408	Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1409	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1410	(uint64_t )&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1411	# else
1412	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1413	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1414	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1415	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1416	pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1417	pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1418	pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1419	pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1420	# endif
1421	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1422	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1423
1424	return u64Ret;
1425
1426	# else /* !IEM_WITH_CODE_TLB */
1427	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1428	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1429	{
1430	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1431	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1432	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1433	# else
1434	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1435	pVCpu->iem.s.abOpcode[offOpcode + 1],
1436	pVCpu->iem.s.abOpcode[offOpcode + 2],
1437	pVCpu->iem.s.abOpcode[offOpcode + 3],
1438	pVCpu->iem.s.abOpcode[offOpcode + 4],
1439	pVCpu->iem.s.abOpcode[offOpcode + 5],
1440	pVCpu->iem.s.abOpcode[offOpcode + 6],
1441	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1442	# endif
1443	}
1444	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1445	# endif /* !IEM_WITH_CODE_TLB */
1446	}
1447
1448	# endif /* IEM_WITH_SETJMP */
1449
1450	/**
1451	* Fetches the next opcode quad word, returns automatically on failure.
1452	*
1453	* @param a_pu64 Where to return the opcode quad word.
1454	* @remark Implicitly references pVCpu.
1455	*/
1456	# ifndef IEM_WITH_SETJMP
1457	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1458	do \
1459	{ \
1460	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1461	if (rcStrict2 != VINF_SUCCESS) \
1462	return rcStrict2; \
1463	} while (0)
1464	# else
1465	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1466	# endif
1467
1468	/**
1469	* For fetching the opcode bytes for an ModR/M effective address, but throw
1470	* away the result.
1471	*
1472	* This is used when decoding undefined opcodes and such where we want to avoid
1473	* unnecessary MC blocks.
1474	*
1475	* @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1476	* used instead. At least for now...
1477	*/
1478	# ifndef IEM_WITH_SETJMP
1479	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1480	RTGCPTR GCPtrEff; \
1481	VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1482	if (rcStrict != VINF_SUCCESS) \
1483	return rcStrict; \
1484	} while (0)
1485	# else
1486	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1487	(void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1488	} while (0)
1489	# endif
1490
1491	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1492
1493
1494	/** @name Misc Worker Functions.
1495	* @{
1496	*/
1497
1498	/**
1499	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1500	* not (kind of obsolete now).
1501	*
1502	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1503	*/
1504	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1505
1506	/**
1507	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1508	*
1509	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1510	* @param a_fEfl The new EFLAGS.
1511	*/
1512	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1513
1514
1515	/**
1516	* Loads a NULL data selector into a selector register, both the hidden and
1517	* visible parts, in protected mode.
1518	*
1519	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1520	* @param pSReg Pointer to the segment register.
1521	* @param uRpl The RPL.
1522	*/
1523	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1524	{
1525	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1526	* data selector in protected mode. */
1527	pSReg->Sel = uRpl;
1528	pSReg->ValidSel = uRpl;
1529	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1530	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1531	{
1532	/* VT-x (Intel 3960x) observed doing something like this. */
1533	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1534	pSReg->u32Limit = UINT32_MAX;
1535	pSReg->u64Base = 0;
1536	}
1537	else
1538	{
1539	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1540	pSReg->u32Limit = 0;
1541	pSReg->u64Base = 0;
1542	}
1543	}
1544
1545	/** @} */
1546
1547
1548	/*
1549	*
1550	* Helpers routines.
1551	* Helpers routines.
1552	* Helpers routines.
1553	*
1554	*/
1555
1556	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1557
1558	/**
1559	* Recalculates the effective operand size.
1560	*
1561	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1562	*/
1563	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1564	{
1565	switch (IEM_GET_CPU_MODE(pVCpu))
1566	{
1567	case IEMMODE_16BIT:
1568	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1569	break;
1570	case IEMMODE_32BIT:
1571	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1572	break;
1573	case IEMMODE_64BIT:
1574	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1575	{
1576	case 0:
1577	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1578	break;
1579	case IEM_OP_PRF_SIZE_OP:
1580	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1581	break;
1582	case IEM_OP_PRF_SIZE_REX_W:
1583	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1584	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1585	break;
1586	}
1587	break;
1588	default:
1589	AssertFailed();
1590	}
1591	}
1592
1593
1594	/**
1595	* Sets the default operand size to 64-bit and recalculates the effective
1596	* operand size.
1597	*
1598	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1599	*/
1600	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1601	{
1602	Assert(IEM_IS_64BIT_CODE(pVCpu));
1603	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1604	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1605	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1606	else
1607	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1608	}
1609
1610
1611	/**
1612	* Sets the default operand size to 64-bit and recalculates the effective
1613	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1614	*
1615	* This is for the relative jumps.
1616	*
1617	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1618	*/
1619	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1620	{
1621	Assert(IEM_IS_64BIT_CODE(pVCpu));
1622	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1623	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1624	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1625	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1626	else
1627	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1628	}
1629
1630	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1631
1632
1633
1634	/** @name Register Access.
1635	* @{
1636	*/
1637
1638	/**
1639	* Gets a reference (pointer) to the specified hidden segment register.
1640	*
1641	* @returns Hidden register reference.
1642	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1643	* @param iSegReg The segment register.
1644	*/
1645	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1646	{
1647	Assert(iSegReg < X86_SREG_COUNT);
1648	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1649	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1650
1651	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1652	return pSReg;
1653	}
1654
1655
1656	/**
1657	* Ensures that the given hidden segment register is up to date.
1658	*
1659	* @returns Hidden register reference.
1660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1661	* @param pSReg The segment register.
1662	*/
1663	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1664	{
1665	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1666	NOREF(pVCpu);
1667	return pSReg;
1668	}
1669
1670
1671	/**
1672	* Gets a reference (pointer) to the specified segment register (the selector
1673	* value).
1674	*
1675	* @returns Pointer to the selector variable.
1676	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1677	* @param iSegReg The segment register.
1678	*/
1679	DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1680	{
1681	Assert(iSegReg < X86_SREG_COUNT);
1682	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1683	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1684	}
1685
1686
1687	/**
1688	* Fetches the selector value of a segment register.
1689	*
1690	* @returns The selector value.
1691	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1692	* @param iSegReg The segment register.
1693	*/
1694	DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1695	{
1696	Assert(iSegReg < X86_SREG_COUNT);
1697	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1698	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1699	}
1700
1701
1702	/**
1703	* Fetches the base address value of a segment register.
1704	*
1705	* @returns The selector value.
1706	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1707	* @param iSegReg The segment register.
1708	*/
1709	DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1710	{
1711	Assert(iSegReg < X86_SREG_COUNT);
1712	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1713	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1714	}
1715
1716
1717	/**
1718	* Gets a reference (pointer) to the specified general purpose register.
1719	*
1720	* @returns Register reference.
1721	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1722	* @param iReg The general purpose register.
1723	*/
1724	DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1725	{
1726	Assert(iReg < 16);
1727	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1728	}
1729
1730
1731	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1732	/**
1733	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1734	*
1735	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1736	*
1737	* @returns Register reference.
1738	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1739	* @param iReg The register.
1740	*/
1741	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1742	{
1743	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1744	{
1745	Assert(iReg < 16);
1746	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1747	}
1748	/* high 8-bit register. */
1749	Assert(iReg < 8);
1750	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1751	}
1752	#endif
1753
1754
1755	/**
1756	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1757	* alternative version with extended (20) register index.
1758	*
1759	* @returns Register reference.
1760	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1761	* @param iRegEx The register. The 16 first are regular ones,
1762	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1763	*/
1764	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1765	{
1766	/** @todo This could be done by double indexing on little endian hosts:
1767	* return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1768	if (iRegEx < 16)
1769	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1770
1771	/* high 8-bit register. */
1772	Assert(iRegEx < 20);
1773	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1774	}
1775
1776
1777	/**
1778	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1779	*
1780	* @returns Register reference.
1781	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1782	* @param iReg The register.
1783	*/
1784	DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1785	{
1786	Assert(iReg < 16);
1787	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1788	}
1789
1790
1791	/**
1792	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1793	*
1794	* @returns Register reference.
1795	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1796	* @param iReg The register.
1797	*/
1798	DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1799	{
1800	Assert(iReg < 16);
1801	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1802	}
1803
1804
1805	/**
1806	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1807	*
1808	* @returns Register reference.
1809	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1810	* @param iReg The register.
1811	*/
1812	DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1813	{
1814	Assert(iReg < 16);
1815	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1816	}
1817
1818
1819	/**
1820	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1821	*
1822	* @returns Register reference.
1823	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1824	* @param iReg The register.
1825	*/
1826	DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1827	{
1828	Assert(iReg < 64);
1829	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1830	}
1831
1832
1833	/**
1834	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1835	*
1836	* @returns Register reference.
1837	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1838	* @param iReg The register.
1839	*/
1840	DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1841	{
1842	Assert(iReg < 16);
1843	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1844	}
1845
1846
1847	/**
1848	* Gets a reference (pointer) to the specified segment register's base address.
1849	*
1850	* @returns Segment register base address reference.
1851	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1852	* @param iSegReg The segment selector.
1853	*/
1854	DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1855	{
1856	Assert(iSegReg < X86_SREG_COUNT);
1857	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1858	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1859	}
1860
1861
1862	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1863	/**
1864	* Fetches the value of a 8-bit general purpose register.
1865	*
1866	* @returns The register value.
1867	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1868	* @param iReg The register.
1869	*/
1870	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1871	{
1872	return *iemGRegRefU8(pVCpu, iReg);
1873	}
1874	#endif
1875
1876
1877	/**
1878	* Fetches the value of a 8-bit general purpose register, alternative version
1879	* with extended (20) register index.
1880
1881	* @returns The register value.
1882	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1883	* @param iRegEx The register. The 16 first are regular ones,
1884	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1885	*/
1886	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1887	{
1888	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1889	}
1890
1891
1892	/**
1893	* Fetches the value of a 16-bit general purpose register.
1894	*
1895	* @returns The register value.
1896	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1897	* @param iReg The register.
1898	*/
1899	DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1900	{
1901	Assert(iReg < 16);
1902	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1903	}
1904
1905
1906	/**
1907	* Fetches the value of a 32-bit general purpose register.
1908	*
1909	* @returns The register value.
1910	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1911	* @param iReg The register.
1912	*/
1913	DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1914	{
1915	Assert(iReg < 16);
1916	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1917	}
1918
1919
1920	/**
1921	* Fetches the value of a 64-bit general purpose register.
1922	*
1923	* @returns The register value.
1924	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1925	* @param iReg The register.
1926	*/
1927	DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1928	{
1929	Assert(iReg < 16);
1930	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1931	}
1932
1933
1934	/**
1935	* Stores a 16-bit value to a general purpose register.
1936	*
1937	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1938	* @param iReg The register.
1939	* @param uValue The value to store.
1940	*/
1941	DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1942	{
1943	Assert(iReg < 16);
1944	pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1945	}
1946
1947
1948	/**
1949	* Stores a 32-bit value to a general purpose register, implicitly clearing high
1950	* values.
1951	*
1952	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1953	* @param iReg The register.
1954	* @param uValue The value to store.
1955	*/
1956	DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1957	{
1958	Assert(iReg < 16);
1959	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1960	}
1961
1962
1963	/**
1964	* Stores a 64-bit value to a general purpose register.
1965	*
1966	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1967	* @param iReg The register.
1968	* @param uValue The value to store.
1969	*/
1970	DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1971	{
1972	Assert(iReg < 16);
1973	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1974	}
1975
1976
1977	/**
1978	* Get the address of the top of the stack.
1979	*
1980	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1981	*/
1982	DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1983	{
1984	if (IEM_IS_64BIT_CODE(pVCpu))
1985	return pVCpu->cpum.GstCtx.rsp;
1986	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1987	return pVCpu->cpum.GstCtx.esp;
1988	return pVCpu->cpum.GstCtx.sp;
1989	}
1990
1991
1992	/**
1993	* Updates the RIP/EIP/IP to point to the next instruction.
1994	*
1995	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1996	* @param cbInstr The number of bytes to add.
1997	*/
1998	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1999	{
2000	/*
2001	* Advance RIP.
2002	*
2003	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
2004	* while in all other modes except LM64 the updates are 32-bit. This means
2005	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
2006	* 4GB and 64KB rollovers, and decide whether anything needs masking.
2007	*
2008	* See PC wrap around tests in bs3-cpu-weird-1.
2009	*/
2010	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2011	uint64_t const uRipNext = uRipPrev + cbInstr;
2012	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
2013	\|\| IEM_IS_64BIT_CODE(pVCpu)))
2014	pVCpu->cpum.GstCtx.rip = uRipNext;
2015	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2016	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2017	else
2018	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2019	}
2020
2021
2022	/**
2023	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2024	* following EFLAGS bits are set:
2025	* - X86_EFL_RF - clear it.
2026	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2027	* - X86_EFL_TF - generate single step \#DB trap.
2028	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2029	* instruction).
2030	*
2031	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2032	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2033	* takes priority over both NMIs and hardware interrupts. So, neither is
2034	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2035	* either unsupported will be triggered on-top of any \#DB raised here.)
2036	*
2037	* The RF flag only needs to be cleared here as it only suppresses instruction
2038	* breakpoints which are not raised here (happens synchronously during
2039	* instruction fetching).
2040	*
2041	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2042	* status has no bearing on whether \#DB exceptions are raised.
2043	*
2044	* @note This must NOT be called by the two instructions setting the
2045	* CPUMCTX_INHIBIT_SHADOW_SS flag.
2046	*
2047	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2048	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2049	* Stacks}
2050	*/
2051	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2052	{
2053	/*
2054	* Normally we're just here to clear RF and/or interrupt shadow bits.
2055	*/
2056	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
2057	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
2058	else
2059	{
2060	/*
2061	* Raise a #DB or/and DBGF event.
2062	*/
2063	VBOXSTRICTRC rcStrict;
2064	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
2065	{
2066	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2067	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2068	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2069	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
2070	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2071	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2072	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2073	pVCpu->cpum.GstCtx.rflags.uBoth));
2074
2075	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
2076	rcStrict = iemRaiseDebugException(pVCpu);
2077
2078	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2079	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2080	{
2081	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2082	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2083	}
2084	}
2085	else
2086	{
2087	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2088	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2089	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2090	}
2091	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2092	return rcStrict;
2093	}
2094	return VINF_SUCCESS;
2095	}
2096
2097
2098	/**
2099	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2100	*
2101	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2102	*/
2103	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2104	{
2105	/*
2106	* We assume that most of the time nothing actually needs doing here.
2107	*/
2108	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2109	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2110	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
2111	return VINF_SUCCESS;
2112	return iemFinishInstructionWithFlagsSet(pVCpu);
2113	}
2114
2115
2116	/**
2117	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2118	* and CPUMCTX_INHIBIT_SHADOW.
2119	*
2120	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2121	* @param cbInstr The number of bytes to add.
2122	*/
2123	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2124	{
2125	iemRegAddToRip(pVCpu, cbInstr);
2126	return iemRegFinishClearingRF(pVCpu);
2127	}
2128
2129
2130	/**
2131	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
2132	* and CPUMCTX_INHIBIT_SHADOW.
2133	*
2134	* Only called from 64-bit code.
2135	*
2136	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2137	* @param cbInstr The number of bytes to add.
2138	*/
2139	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2140	{
2141	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2142	return iemRegFinishClearingRF(pVCpu);
2143	}
2144
2145
2146	/**
2147	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2148	* CPUMCTX_INHIBIT_SHADOW.
2149	*
2150	* This is never from 64-bit code.
2151	*
2152	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2153	* @param cbInstr The number of bytes to add.
2154	*/
2155	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2156	{
2157	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2158	return iemRegFinishClearingRF(pVCpu);
2159	}
2160
2161
2162	/**
2163	* Updates the IP to point to the next instruction and clears EFLAGS.RF and
2164	* CPUMCTX_INHIBIT_SHADOW.
2165	*
2166	* This is only ever used from 16-bit code on a pre-386 CPU.
2167	*
2168	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2169	* @param cbInstr The number of bytes to add.
2170	*/
2171	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2172	{
2173	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2174	return iemRegFinishClearingRF(pVCpu);
2175	}
2176
2177
2178	/**
2179	* Adds a 8-bit signed jump offset to RIP from 64-bit code.
2180	*
2181	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2182	* segment limit.
2183	*
2184	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2185	* @param cbInstr Instruction size.
2186	* @param offNextInstr The offset of the next instruction.
2187	* @param enmEffOpSize Effective operand size.
2188	*/
2189	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2190	IEMMODE enmEffOpSize) RT_NOEXCEPT
2191	{
2192	Assert(IEM_IS_64BIT_CODE(pVCpu));
2193	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2194
2195	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2196	if (enmEffOpSize == IEMMODE_16BIT)
2197	uNewRip &= UINT16_MAX;
2198
2199	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2200	pVCpu->cpum.GstCtx.rip = uNewRip;
2201	else
2202	return iemRaiseGeneralProtectionFault0(pVCpu);
2203
2204	#ifndef IEM_WITH_CODE_TLB
2205	iemOpcodeFlushLight(pVCpu, cbInstr);
2206	#endif
2207
2208	/*
2209	* Clear RF and finish the instruction (maybe raise #DB).
2210	*/
2211	return iemRegFinishClearingRF(pVCpu);
2212	}
2213
2214
2215	/**
2216	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2217	* code (never 64-bit).
2218	*
2219	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2220	* segment limit.
2221	*
2222	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2223	* @param cbInstr Instruction size.
2224	* @param offNextInstr The offset of the next instruction.
2225	* @param enmEffOpSize Effective operand size.
2226	*/
2227	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2228	IEMMODE enmEffOpSize) RT_NOEXCEPT
2229	{
2230	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2231	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2232
2233	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2234	if (enmEffOpSize == IEMMODE_16BIT)
2235	uNewEip &= UINT16_MAX;
2236	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2237	pVCpu->cpum.GstCtx.rip = uNewEip;
2238	else
2239	return iemRaiseGeneralProtectionFault0(pVCpu);
2240
2241	#ifndef IEM_WITH_CODE_TLB
2242	iemOpcodeFlushLight(pVCpu, cbInstr);
2243	#endif
2244
2245	/*
2246	* Clear RF and finish the instruction (maybe raise #DB).
2247	*/
2248	return iemRegFinishClearingRF(pVCpu);
2249	}
2250
2251
2252	/**
2253	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2254	*
2255	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2256	* segment limit.
2257	*
2258	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2259	* @param cbInstr Instruction size.
2260	* @param offNextInstr The offset of the next instruction.
2261	*/
2262	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2263	int8_t offNextInstr) RT_NOEXCEPT
2264	{
2265	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2266
2267	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2268	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2269	pVCpu->cpum.GstCtx.rip = uNewIp;
2270	else
2271	return iemRaiseGeneralProtectionFault0(pVCpu);
2272
2273	#ifndef IEM_WITH_CODE_TLB
2274	iemOpcodeFlushLight(pVCpu, cbInstr);
2275	#endif
2276
2277	/*
2278	* Clear RF and finish the instruction (maybe raise #DB).
2279	*/
2280	return iemRegFinishClearingRF(pVCpu);
2281	}
2282
2283
2284	/**
2285	* Adds a 16-bit signed jump offset to RIP from 64-bit code.
2286	*
2287	* @returns Strict VBox status code.
2288	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2289	* @param cbInstr Instruction size.
2290	* @param offNextInstr The offset of the next instruction.
2291	*/
2292	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2293	int16_t offNextInstr) RT_NOEXCEPT
2294	{
2295	Assert(IEM_IS_64BIT_CODE(pVCpu));
2296
2297	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2298
2299	#ifndef IEM_WITH_CODE_TLB
2300	iemOpcodeFlushLight(pVCpu, cbInstr);
2301	#endif
2302
2303	/*
2304	* Clear RF and finish the instruction (maybe raise #DB).
2305	*/
2306	return iemRegFinishClearingRF(pVCpu);
2307	}
2308
2309
2310	/**
2311	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2312	*
2313	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2314	* segment limit.
2315	*
2316	* @returns Strict VBox status code.
2317	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2318	* @param cbInstr Instruction size.
2319	* @param offNextInstr The offset of the next instruction.
2320	*
2321	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2322	* identical.
2323	*/
2324	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2325	int16_t offNextInstr) RT_NOEXCEPT
2326	{
2327	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2328
2329	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2330	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2331	pVCpu->cpum.GstCtx.rip = uNewIp;
2332	else
2333	return iemRaiseGeneralProtectionFault0(pVCpu);
2334
2335	#ifndef IEM_WITH_CODE_TLB
2336	iemOpcodeFlushLight(pVCpu, cbInstr);
2337	#endif
2338
2339	/*
2340	* Clear RF and finish the instruction (maybe raise #DB).
2341	*/
2342	return iemRegFinishClearingRF(pVCpu);
2343	}
2344
2345
2346	/**
2347	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2348	*
2349	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2350	* segment limit.
2351	*
2352	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2353	* only alternative for relative jumps in 64-bit code and that is already
2354	* handled in the decoder stage.
2355	*
2356	* @returns Strict VBox status code.
2357	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2358	* @param cbInstr Instruction size.
2359	* @param offNextInstr The offset of the next instruction.
2360	*/
2361	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2362	int32_t offNextInstr) RT_NOEXCEPT
2363	{
2364	Assert(IEM_IS_64BIT_CODE(pVCpu));
2365
2366	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2367	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2368	pVCpu->cpum.GstCtx.rip = uNewRip;
2369	else
2370	return iemRaiseGeneralProtectionFault0(pVCpu);
2371
2372	#ifndef IEM_WITH_CODE_TLB
2373	iemOpcodeFlushLight(pVCpu, cbInstr);
2374	#endif
2375
2376	/*
2377	* Clear RF and finish the instruction (maybe raise #DB).
2378	*/
2379	return iemRegFinishClearingRF(pVCpu);
2380	}
2381
2382
2383	/**
2384	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2385	*
2386	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2387	* segment limit.
2388	*
2389	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2390	* only alternative for relative jumps in 32-bit code and that is already
2391	* handled in the decoder stage.
2392	*
2393	* @returns Strict VBox status code.
2394	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2395	* @param cbInstr Instruction size.
2396	* @param offNextInstr The offset of the next instruction.
2397	*/
2398	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2399	int32_t offNextInstr) RT_NOEXCEPT
2400	{
2401	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2402	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2403
2404	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2405	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2406	pVCpu->cpum.GstCtx.rip = uNewEip;
2407	else
2408	return iemRaiseGeneralProtectionFault0(pVCpu);
2409
2410	#ifndef IEM_WITH_CODE_TLB
2411	iemOpcodeFlushLight(pVCpu, cbInstr);
2412	#endif
2413
2414	/*
2415	* Clear RF and finish the instruction (maybe raise #DB).
2416	*/
2417	return iemRegFinishClearingRF(pVCpu);
2418	}
2419
2420
2421	/**
2422	* Extended version of iemFinishInstructionWithFlagsSet that goes with
2423	* iemRegAddToRipAndFinishingClearingRfEx.
2424	*
2425	* See iemFinishInstructionWithFlagsSet() for details.
2426	*/
2427	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2428	{
2429	/*
2430	* Raise a #DB.
2431	*/
2432	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2433	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2434	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
2435	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2436	/** @todo Do we set all pending \#DB events, or just one? */
2437	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2438	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2439	pVCpu->cpum.GstCtx.rflags.uBoth));
2440	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
2441	return iemRaiseDebugException(pVCpu);
2442	}
2443
2444
2445	/**
2446	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2447	* others potentially updating EFLAGS.TF.
2448	*
2449	* The single step event must be generated using the TF value at the start of
2450	* the instruction, not the new value set by it.
2451	*
2452	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2453	* @param cbInstr The number of bytes to add.
2454	* @param fEflOld The EFLAGS at the start of the instruction
2455	* execution.
2456	*/
2457	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2458	{
2459	iemRegAddToRip(pVCpu, cbInstr);
2460	if (!(fEflOld & X86_EFL_TF))
2461	return iemRegFinishClearingRF(pVCpu);
2462	return iemFinishInstructionWithTfSet(pVCpu);
2463	}
2464
2465
2466	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2467	/**
2468	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2469	*
2470	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2471	*/
2472	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2473	{
2474	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2475	}
2476	#endif
2477
2478
2479	/**
2480	* Adds to the stack pointer.
2481	*
2482	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2483	* @param cbToAdd The number of bytes to add (8-bit!).
2484	*/
2485	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2486	{
2487	if (IEM_IS_64BIT_CODE(pVCpu))
2488	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2489	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2490	pVCpu->cpum.GstCtx.esp += cbToAdd;
2491	else
2492	pVCpu->cpum.GstCtx.sp += cbToAdd;
2493	}
2494
2495
2496	/**
2497	* Subtracts from the stack pointer.
2498	*
2499	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2500	* @param cbToSub The number of bytes to subtract (8-bit!).
2501	*/
2502	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2503	{
2504	if (IEM_IS_64BIT_CODE(pVCpu))
2505	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2506	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2507	pVCpu->cpum.GstCtx.esp -= cbToSub;
2508	else
2509	pVCpu->cpum.GstCtx.sp -= cbToSub;
2510	}
2511
2512
2513	/**
2514	* Adds to the temporary stack pointer.
2515	*
2516	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2517	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2518	* @param cbToAdd The number of bytes to add (16-bit).
2519	*/
2520	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2521	{
2522	if (IEM_IS_64BIT_CODE(pVCpu))
2523	pTmpRsp->u += cbToAdd;
2524	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2525	pTmpRsp->DWords.dw0 += cbToAdd;
2526	else
2527	pTmpRsp->Words.w0 += cbToAdd;
2528	}
2529
2530
2531	/**
2532	* Subtracts from the temporary stack pointer.
2533	*
2534	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2535	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2536	* @param cbToSub The number of bytes to subtract.
2537	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2538	* expecting that.
2539	*/
2540	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2541	{
2542	if (IEM_IS_64BIT_CODE(pVCpu))
2543	pTmpRsp->u -= cbToSub;
2544	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2545	pTmpRsp->DWords.dw0 -= cbToSub;
2546	else
2547	pTmpRsp->Words.w0 -= cbToSub;
2548	}
2549
2550
2551	/**
2552	* Calculates the effective stack address for a push of the specified size as
2553	* well as the new RSP value (upper bits may be masked).
2554	*
2555	* @returns Effective stack addressf for the push.
2556	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2557	* @param cbItem The size of the stack item to pop.
2558	* @param puNewRsp Where to return the new RSP value.
2559	*/
2560	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2561	{
2562	RTUINT64U uTmpRsp;
2563	RTGCPTR GCPtrTop;
2564	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2565
2566	if (IEM_IS_64BIT_CODE(pVCpu))
2567	GCPtrTop = uTmpRsp.u -= cbItem;
2568	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2569	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2570	else
2571	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2572	*puNewRsp = uTmpRsp.u;
2573	return GCPtrTop;
2574	}
2575
2576
2577	/**
2578	* Gets the current stack pointer and calculates the value after a pop of the
2579	* specified size.
2580	*
2581	* @returns Current stack pointer.
2582	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2583	* @param cbItem The size of the stack item to pop.
2584	* @param puNewRsp Where to return the new RSP value.
2585	*/
2586	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2587	{
2588	RTUINT64U uTmpRsp;
2589	RTGCPTR GCPtrTop;
2590	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2591
2592	if (IEM_IS_64BIT_CODE(pVCpu))
2593	{
2594	GCPtrTop = uTmpRsp.u;
2595	uTmpRsp.u += cbItem;
2596	}
2597	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2598	{
2599	GCPtrTop = uTmpRsp.DWords.dw0;
2600	uTmpRsp.DWords.dw0 += cbItem;
2601	}
2602	else
2603	{
2604	GCPtrTop = uTmpRsp.Words.w0;
2605	uTmpRsp.Words.w0 += cbItem;
2606	}
2607	*puNewRsp = uTmpRsp.u;
2608	return GCPtrTop;
2609	}
2610
2611
2612	/**
2613	* Calculates the effective stack address for a push of the specified size as
2614	* well as the new temporary RSP value (upper bits may be masked).
2615	*
2616	* @returns Effective stack addressf for the push.
2617	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2618	* @param pTmpRsp The temporary stack pointer. This is updated.
2619	* @param cbItem The size of the stack item to pop.
2620	*/
2621	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2622	{
2623	RTGCPTR GCPtrTop;
2624
2625	if (IEM_IS_64BIT_CODE(pVCpu))
2626	GCPtrTop = pTmpRsp->u -= cbItem;
2627	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2628	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2629	else
2630	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2631	return GCPtrTop;
2632	}
2633
2634
2635	/**
2636	* Gets the effective stack address for a pop of the specified size and
2637	* calculates and updates the temporary RSP.
2638	*
2639	* @returns Current stack pointer.
2640	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2641	* @param pTmpRsp The temporary stack pointer. This is updated.
2642	* @param cbItem The size of the stack item to pop.
2643	*/
2644	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2645	{
2646	RTGCPTR GCPtrTop;
2647	if (IEM_IS_64BIT_CODE(pVCpu))
2648	{
2649	GCPtrTop = pTmpRsp->u;
2650	pTmpRsp->u += cbItem;
2651	}
2652	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2653	{
2654	GCPtrTop = pTmpRsp->DWords.dw0;
2655	pTmpRsp->DWords.dw0 += cbItem;
2656	}
2657	else
2658	{
2659	GCPtrTop = pTmpRsp->Words.w0;
2660	pTmpRsp->Words.w0 += cbItem;
2661	}
2662	return GCPtrTop;
2663	}
2664
2665	/** @} */
2666
2667
2668	/** @name FPU access and helpers.
2669	*
2670	* @{
2671	*/
2672
2673
2674	/**
2675	* Hook for preparing to use the host FPU.
2676	*
2677	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2678	*
2679	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2680	*/
2681	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2682	{
2683	#ifdef IN_RING3
2684	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2685	#else
2686	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2687	#endif
2688	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2689	}
2690
2691
2692	/**
2693	* Hook for preparing to use the host FPU for SSE.
2694	*
2695	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2696	*
2697	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2698	*/
2699	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2700	{
2701	iemFpuPrepareUsage(pVCpu);
2702	}
2703
2704
2705	/**
2706	* Hook for preparing to use the host FPU for AVX.
2707	*
2708	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2709	*
2710	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2711	*/
2712	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2713	{
2714	iemFpuPrepareUsage(pVCpu);
2715	}
2716
2717
2718	/**
2719	* Hook for actualizing the guest FPU state before the interpreter reads it.
2720	*
2721	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2722	*
2723	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2724	*/
2725	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2726	{
2727	#ifdef IN_RING3
2728	NOREF(pVCpu);
2729	#else
2730	CPUMRZFpuStateActualizeForRead(pVCpu);
2731	#endif
2732	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2733	}
2734
2735
2736	/**
2737	* Hook for actualizing the guest FPU state before the interpreter changes it.
2738	*
2739	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2740	*
2741	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2742	*/
2743	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2744	{
2745	#ifdef IN_RING3
2746	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2747	#else
2748	CPUMRZFpuStateActualizeForChange(pVCpu);
2749	#endif
2750	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2751	}
2752
2753
2754	/**
2755	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2756	* only.
2757	*
2758	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2759	*
2760	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2761	*/
2762	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2763	{
2764	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2765	NOREF(pVCpu);
2766	#else
2767	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2768	#endif
2769	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2770	}
2771
2772
2773	/**
2774	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2775	* read+write.
2776	*
2777	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2778	*
2779	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2780	*/
2781	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2782	{
2783	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2784	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2785	#else
2786	CPUMRZFpuStateActualizeForChange(pVCpu);
2787	#endif
2788	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2789
2790	/* Make sure any changes are loaded the next time around. */
2791	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2792	}
2793
2794
2795	/**
2796	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2797	* only.
2798	*
2799	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2800	*
2801	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2802	*/
2803	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2804	{
2805	#ifdef IN_RING3
2806	NOREF(pVCpu);
2807	#else
2808	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2809	#endif
2810	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2811	}
2812
2813
2814	/**
2815	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2816	* read+write.
2817	*
2818	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2819	*
2820	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2821	*/
2822	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2823	{
2824	#ifdef IN_RING3
2825	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2826	#else
2827	CPUMRZFpuStateActualizeForChange(pVCpu);
2828	#endif
2829	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2830
2831	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2832	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2833	}
2834
2835
2836	/**
2837	* Stores a QNaN value into a FPU register.
2838	*
2839	* @param pReg Pointer to the register.
2840	*/
2841	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2842	{
2843	pReg->au32[0] = UINT32_C(0x00000000);
2844	pReg->au32[1] = UINT32_C(0xc0000000);
2845	pReg->au16[4] = UINT16_C(0xffff);
2846	}
2847
2848
2849	/**
2850	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
2851	*
2852	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2853	* @param pFpuCtx The FPU context.
2854	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2855	*/
2856	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2857	{
2858	Assert(uFpuOpcode != UINT16_MAX);
2859	pFpuCtx->FOP = uFpuOpcode;
2860	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2861	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2862	{
2863	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2864	* happens in real mode here based on the fnsave and fnstenv images. */
2865	pFpuCtx->CS = 0;
2866	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2867	}
2868	else if (!IEM_IS_LONG_MODE(pVCpu))
2869	{
2870	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2871	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2872	}
2873	else
2874	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2875	}
2876
2877
2878	/**
2879	* Marks the specified stack register as free (for FFREE).
2880	*
2881	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2882	* @param iStReg The register to free.
2883	*/
2884	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2885	{
2886	Assert(iStReg < 8);
2887	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2888	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2889	pFpuCtx->FTW &= ~RT_BIT(iReg);
2890	}
2891
2892
2893	/**
2894	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2895	*
2896	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2897	*/
2898	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2899	{
2900	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2901	uint16_t uFsw = pFpuCtx->FSW;
2902	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2903	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2904	uFsw &= ~X86_FSW_TOP_MASK;
2905	uFsw \|= uTop;
2906	pFpuCtx->FSW = uFsw;
2907	}
2908
2909
2910	/**
2911	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2912	*
2913	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2914	*/
2915	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2916	{
2917	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2918	uint16_t uFsw = pFpuCtx->FSW;
2919	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2920	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2921	uFsw &= ~X86_FSW_TOP_MASK;
2922	uFsw \|= uTop;
2923	pFpuCtx->FSW = uFsw;
2924	}
2925
2926
2927
2928
2929	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2930	{
2931	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2932	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2933	if (pFpuCtx->FTW & RT_BIT(iReg))
2934	return VINF_SUCCESS;
2935	return VERR_NOT_FOUND;
2936	}
2937
2938
2939	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2940	{
2941	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2942	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2943	if (pFpuCtx->FTW & RT_BIT(iReg))
2944	{
2945	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2946	return VINF_SUCCESS;
2947	}
2948	return VERR_NOT_FOUND;
2949	}
2950
2951
2952	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2953	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2954	{
2955	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2956	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2957	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2958	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2959	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2960	{
2961	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2962	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2963	return VINF_SUCCESS;
2964	}
2965	return VERR_NOT_FOUND;
2966	}
2967
2968
2969	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2970	{
2971	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2972	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2973	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2974	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2975	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2976	{
2977	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2978	return VINF_SUCCESS;
2979	}
2980	return VERR_NOT_FOUND;
2981	}
2982
2983
2984	/**
2985	* Rotates the stack registers when setting new TOS.
2986	*
2987	* @param pFpuCtx The FPU context.
2988	* @param iNewTop New TOS value.
2989	* @remarks We only do this to speed up fxsave/fxrstor which
2990	* arrange the FP registers in stack order.
2991	* MUST be done before writing the new TOS (FSW).
2992	*/
2993	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2994	{
2995	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2996	RTFLOAT80U ar80Temp[8];
2997
2998	if (iOldTop == iNewTop)
2999	return;
3000
3001	/* Unscrew the stack and get it into 'native' order. */
3002	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
3003	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
3004	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
3005	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
3006	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
3007	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
3008	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
3009	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3010
3011	/* Now rotate the stack to the new position. */
3012	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3013	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3014	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3015	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3016	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3017	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3018	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3019	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3020	}
3021
3022
3023	/**
3024	* Updates the FPU exception status after FCW is changed.
3025	*
3026	* @param pFpuCtx The FPU context.
3027	*/
3028	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3029	{
3030	uint16_t u16Fsw = pFpuCtx->FSW;
3031	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3032	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
3033	else
3034	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
3035	pFpuCtx->FSW = u16Fsw;
3036	}
3037
3038
3039	/**
3040	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3041	*
3042	* @returns The full FTW.
3043	* @param pFpuCtx The FPU context.
3044	*/
3045	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3046	{
3047	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3048	uint16_t u16Ftw = 0;
3049	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3050	for (unsigned iSt = 0; iSt < 8; iSt++)
3051	{
3052	unsigned const iReg = (iSt + iTop) & 7;
3053	if (!(u8Ftw & RT_BIT(iReg)))
3054	u16Ftw \|= 3 << (iReg * 2); /* empty */
3055	else
3056	{
3057	uint16_t uTag;
3058	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3059	if (pr80Reg->s.uExponent == 0x7fff)
3060	uTag = 2; /* Exponent is all 1's => Special. */
3061	else if (pr80Reg->s.uExponent == 0x0000)
3062	{
3063	if (pr80Reg->s.uMantissa == 0x0000)
3064	uTag = 1; /* All bits are zero => Zero. */
3065	else
3066	uTag = 2; /* Must be special. */
3067	}
3068	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3069	uTag = 0; /* Valid. */
3070	else
3071	uTag = 2; /* Must be special. */
3072
3073	u16Ftw \|= uTag << (iReg * 2);
3074	}
3075	}
3076
3077	return u16Ftw;
3078	}
3079
3080
3081	/**
3082	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3083	*
3084	* @returns The compressed FTW.
3085	* @param u16FullFtw The full FTW to convert.
3086	*/
3087	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3088	{
3089	uint8_t u8Ftw = 0;
3090	for (unsigned i = 0; i < 8; i++)
3091	{
3092	if ((u16FullFtw & 3) != 3 /empty/)
3093	u8Ftw \|= RT_BIT(i);
3094	u16FullFtw >>= 2;
3095	}
3096
3097	return u8Ftw;
3098	}
3099
3100	/** @} */
3101
3102
3103	/** @name Memory access.
3104	*
3105	* @{
3106	*/
3107
3108
3109	/**
3110	* Checks whether alignment checks are enabled or not.
3111	*
3112	* @returns true if enabled, false if not.
3113	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3114	*/
3115	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3116	{
3117	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3118	return IEM_GET_CPL(pVCpu) == 3
3119	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3120	}
3121
3122	/**
3123	* Checks if the given segment can be written to, raise the appropriate
3124	* exception if not.
3125	*
3126	* @returns VBox strict status code.
3127	*
3128	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3129	* @param pHid Pointer to the hidden register.
3130	* @param iSegReg The register number.
3131	* @param pu64BaseAddr Where to return the base address to use for the
3132	* segment. (In 64-bit code it may differ from the
3133	* base in the hidden segment.)
3134	*/
3135	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3136	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3137	{
3138	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3139
3140	if (IEM_IS_64BIT_CODE(pVCpu))
3141	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3142	else
3143	{
3144	if (!pHid->Attr.n.u1Present)
3145	{
3146	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3147	AssertRelease(uSel == 0);
3148	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3149	return iemRaiseGeneralProtectionFault0(pVCpu);
3150	}
3151
3152	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3153	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3154	&& !IEM_IS_64BIT_CODE(pVCpu) )
3155	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3156	*pu64BaseAddr = pHid->u64Base;
3157	}
3158	return VINF_SUCCESS;
3159	}
3160
3161
3162	/**
3163	* Checks if the given segment can be read from, raise the appropriate
3164	* exception if not.
3165	*
3166	* @returns VBox strict status code.
3167	*
3168	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3169	* @param pHid Pointer to the hidden register.
3170	* @param iSegReg The register number.
3171	* @param pu64BaseAddr Where to return the base address to use for the
3172	* segment. (In 64-bit code it may differ from the
3173	* base in the hidden segment.)
3174	*/
3175	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3176	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3177	{
3178	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3179
3180	if (IEM_IS_64BIT_CODE(pVCpu))
3181	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3182	else
3183	{
3184	if (!pHid->Attr.n.u1Present)
3185	{
3186	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3187	AssertRelease(uSel == 0);
3188	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3189	return iemRaiseGeneralProtectionFault0(pVCpu);
3190	}
3191
3192	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3193	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3194	*pu64BaseAddr = pHid->u64Base;
3195	}
3196	return VINF_SUCCESS;
3197	}
3198
3199
3200	/**
3201	* Maps a physical page.
3202	*
3203	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3204	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3205	* @param GCPhysMem The physical address.
3206	* @param fAccess The intended access.
3207	* @param ppvMem Where to return the mapping address.
3208	* @param pLock The PGM lock.
3209	*/
3210	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3211	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3212	{
3213	#ifdef IEM_LOG_MEMORY_WRITES
3214	if (fAccess & IEM_ACCESS_TYPE_WRITE)
3215	return VERR_PGM_PHYS_TLB_CATCH_ALL;
3216	#endif
3217
3218	/** @todo This API may require some improving later. A private deal with PGM
3219	* regarding locking and unlocking needs to be struct. A couple of TLBs
3220	* living in PGM, but with publicly accessible inlined access methods
3221	* could perhaps be an even better solution. */
3222	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3223	GCPhysMem,
3224	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3225	RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3226	ppvMem,
3227	pLock);
3228	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
3229	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3230
3231	return rc;
3232	}
3233
3234
3235	/**
3236	* Unmap a page previously mapped by iemMemPageMap.
3237	*
3238	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3239	* @param GCPhysMem The physical address.
3240	* @param fAccess The intended access.
3241	* @param pvMem What iemMemPageMap returned.
3242	* @param pLock The PGM lock.
3243	*/
3244	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3245	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3246	{
3247	NOREF(pVCpu);
3248	NOREF(GCPhysMem);
3249	NOREF(fAccess);
3250	NOREF(pvMem);
3251	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3252	}
3253
3254	#ifdef IEM_WITH_SETJMP
3255
3256	/** @todo slim this down */
3257	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3258	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3259	{
3260	Assert(cbMem >= 1);
3261	Assert(iSegReg < X86_SREG_COUNT);
3262
3263	/*
3264	* 64-bit mode is simpler.
3265	*/
3266	if (IEM_IS_64BIT_CODE(pVCpu))
3267	{
3268	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3269	{
3270	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3271	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3272	GCPtrMem += pSel->u64Base;
3273	}
3274
3275	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3276	return GCPtrMem;
3277	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3278	}
3279	/*
3280	* 16-bit and 32-bit segmentation.
3281	*/
3282	else if (iSegReg != UINT8_MAX)
3283	{
3284	/** @todo Does this apply to segments with 4G-1 limit? */
3285	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3286	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3287	{
3288	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3289	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3290	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3291	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
3292	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
3293	\| X86_SEL_TYPE_CODE))
3294	{
3295	case X86DESCATTR_P: /* readonly data, expand up */
3296	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
3297	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
3298	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
3299	/* expand up */
3300	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3301	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3302	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3303	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3304	break;
3305
3306	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3307	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
3308	/* expand down */
3309	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3310	&& ( pSel->Attr.n.u1DefBig
3311	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
3312	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3313	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3314	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3315	break;
3316
3317	default:
3318	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3319	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3320	break;
3321	}
3322	}
3323	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3324	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3325	}
3326	/*
3327	* 32-bit flat address.
3328	*/
3329	else
3330	return GCPtrMem;
3331	}
3332
3333
3334	/** @todo slim this down */
3335	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3336	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3337	{
3338	Assert(cbMem >= 1);
3339	Assert(iSegReg < X86_SREG_COUNT);
3340
3341	/*
3342	* 64-bit mode is simpler.
3343	*/
3344	if (IEM_IS_64BIT_CODE(pVCpu))
3345	{
3346	if (iSegReg >= X86_SREG_FS)
3347	{
3348	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3349	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3350	GCPtrMem += pSel->u64Base;
3351	}
3352
3353	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3354	return GCPtrMem;
3355	}
3356	/*
3357	* 16-bit and 32-bit segmentation.
3358	*/
3359	else
3360	{
3361	Assert(GCPtrMem <= UINT32_MAX);
3362	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3363	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3364	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3365	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
3366	if ( fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE) /* data, expand up */
3367	/** @todo explore exactly how the CS stuff works in real mode. See also
3368	* http://www.rcollins.org/Productivity/DescriptorCache.html and
3369	* http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */
3370	\|\| (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. / /* @todo testcase! */
3371	{
3372	/* expand up */
3373	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3374	if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit
3375	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3376	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3377	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3378	}
3379	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
3380	{
3381	/* expand down - the uppger boundary is defined by the B bit, not G. */
3382	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3383	if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit
3384	&& (pSel->Attr.n.u1DefBig \|\| GCPtrLast32 <= UINT32_C(0xffff))
3385	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3386	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3387	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3388	}
3389	else
3390	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3391	}
3392	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3393	}
3394
3395	#endif /* IEM_WITH_SETJMP */
3396
3397	/**
3398	* Fakes a long mode stack selector for SS = 0.
3399	*
3400	* @param pDescSs Where to return the fake stack descriptor.
3401	* @param uDpl The DPL we want.
3402	*/
3403	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3404	{
3405	pDescSs->Long.au64[0] = 0;
3406	pDescSs->Long.au64[1] = 0;
3407	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3408	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3409	pDescSs->Long.Gen.u2Dpl = uDpl;
3410	pDescSs->Long.Gen.u1Present = 1;
3411	pDescSs->Long.Gen.u1Long = 1;
3412	}
3413
3414	/*
3415	* Instantiate R/W inline templates.
3416	*/
3417	#define TMPL_MEM_TYPE uint8_t
3418	#define TMPL_MEM_TYPE_ALIGN 0
3419	#define TMPL_MEM_TYPE_SIZE 1
3420	#define TMPL_MEM_FN_SUFF U8
3421	#define TMPL_MEM_FMT_TYPE "%#04x"
3422	#define TMPL_MEM_FMT_DESC "byte"
3423	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3424
3425	#define TMPL_MEM_TYPE uint16_t
3426	#define TMPL_MEM_TYPE_ALIGN 1
3427	#define TMPL_MEM_TYPE_SIZE 2
3428	#define TMPL_MEM_FN_SUFF U16
3429	#define TMPL_MEM_FMT_TYPE "%#06x"
3430	#define TMPL_MEM_FMT_DESC "word"
3431	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3432
3433	#define TMPL_MEM_TYPE uint32_t
3434	#define TMPL_MEM_TYPE_ALIGN 3
3435	#define TMPL_MEM_TYPE_SIZE 4
3436	#define TMPL_MEM_FN_SUFF U32
3437	#define TMPL_MEM_FMT_TYPE "%#010x"
3438	#define TMPL_MEM_FMT_DESC "dword"
3439	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3440
3441	#define TMPL_MEM_TYPE uint64_t
3442	#define TMPL_MEM_TYPE_ALIGN 7
3443	#define TMPL_MEM_TYPE_SIZE 8
3444	#define TMPL_MEM_FN_SUFF U64
3445	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3446	#define TMPL_MEM_FMT_DESC "qword"
3447	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3448
3449	#define TMPL_MEM_NO_STORE
3450	#define TMPL_MEM_NO_MAPPING
3451	#define TMPL_MEM_TYPE uint64_t
3452	#define TMPL_MEM_TYPE_ALIGN 15
3453	#define TMPL_MEM_TYPE_SIZE 8
3454	#define TMPL_MEM_FN_SUFF U64AlignedU128
3455	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3456	#define TMPL_MEM_FMT_DESC "qword"
3457	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3458
3459	/** @} */
3460
3461
3462	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3463
3464	/**
3465	* Gets CR0 fixed-0 bits in VMX operation.
3466	*
3467	* We do this rather than fetching what we report to the guest (in
3468	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3469	* values regardless of whether unrestricted-guest feature is available on the CPU.
3470	*
3471	* @returns CR0 fixed-0 bits.
3472	* @param pVCpu The cross context virtual CPU structure.
3473	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3474	* must be returned. When @c false, the CR0 fixed-0
3475	* bits for VMX root mode is returned.
3476	*
3477	*/
3478	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3479	{
3480	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3481
3482	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3483	if ( fVmxNonRootMode
3484	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3485	return VMX_V_CR0_FIXED0_UX;
3486	return VMX_V_CR0_FIXED0;
3487	}
3488
3489
3490	/**
3491	* Sets virtual-APIC write emulation as pending.
3492	*
3493	* @param pVCpu The cross context virtual CPU structure.
3494	* @param offApic The offset in the virtual-APIC page that was written.
3495	*/
3496	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3497	{
3498	Assert(offApic < XAPIC_OFF_END + 4);
3499
3500	/*
3501	* Record the currently updated APIC offset, as we need this later for figuring
3502	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
3503	* as for supplying the exit qualification when causing an APIC-write VM-exit.
3504	*/
3505	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3506
3507	/*
3508	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3509	* virtualization or APIC-write emulation).
3510	*/
3511	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3512	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3513	}
3514
3515	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3516
3517	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 100830

Download in other formats: