IEMInline.h@ 100059

Last change on this file since 100059 was 100059, checked in by vboxsync, 18 months ago
VMM/IEM: Use fExec for more tests. bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 109.1 KB

Line
1	/* $Id: IEMInline.h 100059 2023-06-03 00:17:25Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
168	*
169	* Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
170	* reject expand down data segments and conforming code segments.
171	*
172	* ASSUMES that the CPU is in 32-bit mode.
173	*
174	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
175	* @param pVCpu The cross context virtual CPU structure of the
176	* calling thread.
177	* @sa iemCalc32BitFlatIndicatorEsDs
178	*/
179	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
180	{
181	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
182	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
183	\| pVCpu->cpum.GstCtx.cs.Attr.u
184	\| pVCpu->cpum.GstCtx.ss.Attr.u
185	\| pVCpu->cpum.GstCtx.ds.Attr.u)
186	& (X86_SEL_TYPE_ACCESSED \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P))
187	== (X86_SEL_TYPE_ACCESSED \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
188	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
189	\| (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
190	\| (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
191	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
192	== 0
193	&& ( pVCpu->cpum.GstCtx.es.u64Base
194	\| pVCpu->cpum.GstCtx.cs.u64Base
195	\| pVCpu->cpum.GstCtx.ss.u64Base
196	\| pVCpu->cpum.GstCtx.ds.u64Base)
197	== 0
198	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
199	? IEM_F_MODE_X86_32BIT_FLAT : 0;
200	}
201
202
203	/**
204	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
205	* flat already.
206	*
207	* This is used by sysenter.
208	*
209	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
210	* @param pVCpu The cross context virtual CPU structure of the
211	* calling thread.
212	* @sa iemCalc32BitFlatIndicator
213	*/
214	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
215	{
216	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
217	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
218	\| pVCpu->cpum.GstCtx.ds.Attr.u)
219	& (X86_SEL_TYPE_ACCESSED \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P))
220	== (X86_SEL_TYPE_ACCESSED \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
221	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
222	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
223	== 0
224	&& ( pVCpu->cpum.GstCtx.es.u64Base
225	\| pVCpu->cpum.GstCtx.ds.u64Base)
226	== 0
227	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
228	? IEM_F_MODE_X86_32BIT_FLAT : 0;
229	}
230
231
232	/**
233	* Calculates the IEM_F_MODE_XXX and CPL flags.
234	*
235	* @returns IEM_F_MODE_XXX
236	* @param pVCpu The cross context virtual CPU structure of the
237	* calling thread.
238	*/
239	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
240	{
241	/*
242	* We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
243	* here to try get this done as efficiently as possible.
244	*/
245	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
246
247	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
248	{
249	if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
250	{
251	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
252	uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
253	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
254	{
255	Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
256	fExec \|= IEM_F_MODE_X86_32BIT_PROT \| iemCalc32BitFlatIndicator(pVCpu);
257	}
258	else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
259	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
260	fExec \|= IEM_F_MODE_X86_64BIT;
261	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
262	fExec \|= IEM_F_MODE_X86_16BIT_PROT;
263	else
264	fExec \|= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
265	return fExec;
266	}
267	return IEM_F_MODE_X86_16BIT_PROT_V86 \| (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
268	}
269
270	/* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
271	if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
272	{
273	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
274	return IEM_F_MODE_X86_16BIT;
275	return IEM_F_MODE_X86_16BIT_PRE_386;
276	}
277
278	/* 32-bit unreal mode. */
279	return IEM_F_MODE_X86_32BIT \| iemCalc32BitFlatIndicator(pVCpu);
280	}
281
282
283	/**
284	* Calculates the AMD-V and VT-x related context flags.
285	*
286	* @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
287	* IEM_F_X86_CTX_VMX.
288	* @param pVCpu The cross context virtual CPU structure of the
289	* calling thread.
290	*/
291	DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
292	{
293	/*
294	* This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
295	* and CPUMIsGuestInNestedHwvirtMode to some extent.
296	*/
297	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
298
299	AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
300	uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
301	\| (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
302	if (RT_LIKELY(!fTmp))
303	return 0; /* likely */
304
305	if (fTmp & X86_CR4_VMXE)
306	{
307	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
308	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
309	return IEM_F_X86_CTX_VMX \| IEM_F_X86_CTX_IN_GUEST;
310	return IEM_F_X86_CTX_VMX;
311	}
312
313	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
314	if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
315	return IEM_F_X86_CTX_SVM \| IEM_F_X86_CTX_IN_GUEST;
316	return IEM_F_X86_CTX_SVM;
317	}
318
319
320	/**
321	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
322	*
323	* @returns IEM_F_BRK_PENDING_XXX or zero.
324	* @param pVCpu The cross context virtual CPU structure of the
325	* calling thread.
326	*/
327	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
328	{
329	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
330
331	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
332	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
333	return 0;
334	return iemCalcExecDbgFlagsSlow(pVCpu);
335	}
336
337	/**
338	* Calculates the the IEM_F_XXX flags.
339	*
340	* @returns IEM_F_XXX combination match the current CPU state.
341	* @param pVCpu The cross context virtual CPU structure of the
342	* calling thread.
343	*/
344	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
345	{
346	return iemCalcExecModeAndCplFlags(pVCpu)
347	\| iemCalcExecHwVirtFlags(pVCpu)
348	/* SMM is not yet implemented */
349	\| iemCalcExecDbgFlags(pVCpu)
350	;
351	}
352
353
354	/**
355	* Re-calculates the MODE and CPL parts of IEMCPU::fExec.
356	*
357	* @param pVCpu The cross context virtual CPU structure of the
358	* calling thread.
359	*/
360	DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
361	{
362	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
363	\| iemCalcExecModeAndCplFlags(pVCpu);
364	}
365
366
367	/**
368	* Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
369	*
370	* @param pVCpu The cross context virtual CPU structure of the
371	* calling thread.
372	*/
373	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
374	{
375	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
376	\| iemCalcExecDbgFlags(pVCpu);
377	}
378
379
380	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
381
382	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
383	/**
384	* Initializes the execution state.
385	*
386	* @param pVCpu The cross context virtual CPU structure of the
387	* calling thread.
388	* @param fExecOpts Optional execution flags:
389	* - IEM_F_BYPASS_HANDLERS
390	* - IEM_F_X86_DISREGARD_LOCK
391	*
392	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
393	* side-effects in strict builds.
394	*/
395	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
396	{
397	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
398	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
399	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
400	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
401	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
402	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
403	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
404	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
405	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
406	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
407
408	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
409	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
410	pVCpu->iem.s.cActiveMappings = 0;
411	pVCpu->iem.s.iNextMapping = 0;
412
413	# ifdef VBOX_STRICT
414	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
415	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
416	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
417	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
418	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
419	pVCpu->iem.s.uRexReg = 127;
420	pVCpu->iem.s.uRexB = 127;
421	pVCpu->iem.s.offModRm = 127;
422	pVCpu->iem.s.uRexIndex = 127;
423	pVCpu->iem.s.iEffSeg = 127;
424	pVCpu->iem.s.idxPrefix = 127;
425	pVCpu->iem.s.uVex3rdReg = 127;
426	pVCpu->iem.s.uVexLength = 127;
427	pVCpu->iem.s.fEvexStuff = 127;
428	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
429	# ifdef IEM_WITH_CODE_TLB
430	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
431	pVCpu->iem.s.pbInstrBuf = NULL;
432	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
433	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
434	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
435	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
436	# else
437	pVCpu->iem.s.offOpcode = 127;
438	pVCpu->iem.s.cbOpcode = 127;
439	# endif
440	# endif /* VBOX_STRICT */
441	}
442	# endif /* VBOX_INCLUDED_vmm_dbgf_h */
443
444
445	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
446	/**
447	* Performs a minimal reinitialization of the execution state.
448	*
449	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
450	* 'world-switch' types operations on the CPU. Currently only nested
451	* hardware-virtualization uses it.
452	*
453	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
454	* @param cbInstr The instruction length (for flushing).
455	*/
456	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
457	{
458	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
459	iemOpcodeFlushHeavy(pVCpu, cbInstr);
460	}
461	# endif
462
463
464	/**
465	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
466	*
467	* @param pVCpu The cross context virtual CPU structure of the
468	* calling thread.
469	*/
470	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
471	{
472	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
473	# ifdef VBOX_STRICT
474	# ifdef IEM_WITH_CODE_TLB
475	NOREF(pVCpu);
476	# else
477	pVCpu->iem.s.cbOpcode = 0;
478	# endif
479	# else
480	NOREF(pVCpu);
481	# endif
482	}
483
484
485	/**
486	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
487	*
488	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
489	*
490	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
491	* @param pVCpu The cross context virtual CPU structure of the calling thread.
492	* @param rcStrict The status code to fiddle.
493	*/
494	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
495	{
496	iemUninitExec(pVCpu);
497	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
498	}
499
500
501	/**
502	* Macro used by the IEMExec* method to check the given instruction length.
503	*
504	* Will return on failure!
505	*
506	* @param a_cbInstr The given instruction length.
507	* @param a_cbMin The minimum length.
508	*/
509	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
510	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
511	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
512
513
514	# ifndef IEM_WITH_SETJMP
515
516	/**
517	* Fetches the first opcode byte.
518	*
519	* @returns Strict VBox status code.
520	* @param pVCpu The cross context virtual CPU structure of the
521	* calling thread.
522	* @param pu8 Where to return the opcode byte.
523	*/
524	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
525	{
526	/*
527	* Check for hardware instruction breakpoints.
528	*/
529	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
530	{ /* likely */ }
531	else
532	{
533	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
534	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
535	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
536	{ /* likely */ }
537	else if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
538	return iemRaiseDebugException(pVCpu);
539	else
540	return rcStrict;
541	}
542
543	/*
544	* Fetch the first opcode byte.
545	*/
546	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
547	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
548	{
549	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
550	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
551	return VINF_SUCCESS;
552	}
553	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
554	}
555
556	# else /* IEM_WITH_SETJMP */
557
558	/**
559	* Fetches the first opcode byte, longjmp on error.
560	*
561	* @returns The opcode byte.
562	* @param pVCpu The cross context virtual CPU structure of the calling thread.
563	*/
564	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
565	{
566	/*
567	* Check for hardware instruction breakpoints.
568	*/
569	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
570	{ /* likely */ }
571	else
572	{
573	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
574	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
575	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
576	{ /* likely */ }
577	else
578	{
579	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
580	rcStrict = iemRaiseDebugException(pVCpu);
581	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
582	}
583	}
584
585	/*
586	* Fetch the first opcode byte.
587	*/
588	# ifdef IEM_WITH_CODE_TLB
589	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
590	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
591	if (RT_LIKELY( pbBuf != NULL
592	&& offBuf < pVCpu->iem.s.cbInstrBuf))
593	{
594	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
595	return pbBuf[offBuf];
596	}
597	# else
598	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
599	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
600	{
601	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
602	return pVCpu->iem.s.abOpcode[offOpcode];
603	}
604	# endif
605	return iemOpcodeGetNextU8SlowJmp(pVCpu);
606	}
607
608	# endif /* IEM_WITH_SETJMP */
609
610	/**
611	* Fetches the first opcode byte, returns/throws automatically on failure.
612	*
613	* @param a_pu8 Where to return the opcode byte.
614	* @remark Implicitly references pVCpu.
615	*/
616	# ifndef IEM_WITH_SETJMP
617	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
618	do \
619	{ \
620	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
621	if (rcStrict2 == VINF_SUCCESS) \
622	{ /* likely */ } \
623	else \
624	return rcStrict2; \
625	} while (0)
626	# else
627	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
628	# endif /* IEM_WITH_SETJMP */
629
630
631	# ifndef IEM_WITH_SETJMP
632
633	/**
634	* Fetches the next opcode byte.
635	*
636	* @returns Strict VBox status code.
637	* @param pVCpu The cross context virtual CPU structure of the
638	* calling thread.
639	* @param pu8 Where to return the opcode byte.
640	*/
641	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
642	{
643	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
644	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
645	{
646	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
647	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
648	return VINF_SUCCESS;
649	}
650	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
651	}
652
653	# else /* IEM_WITH_SETJMP */
654
655	/**
656	* Fetches the next opcode byte, longjmp on error.
657	*
658	* @returns The opcode byte.
659	* @param pVCpu The cross context virtual CPU structure of the calling thread.
660	*/
661	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
662	{
663	# ifdef IEM_WITH_CODE_TLB
664	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
665	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
666	if (RT_LIKELY( pbBuf != NULL
667	&& offBuf < pVCpu->iem.s.cbInstrBuf))
668	{
669	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
670	return pbBuf[offBuf];
671	}
672	# else
673	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
674	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
675	{
676	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
677	return pVCpu->iem.s.abOpcode[offOpcode];
678	}
679	# endif
680	return iemOpcodeGetNextU8SlowJmp(pVCpu);
681	}
682
683	# endif /* IEM_WITH_SETJMP */
684
685	/**
686	* Fetches the next opcode byte, returns automatically on failure.
687	*
688	* @param a_pu8 Where to return the opcode byte.
689	* @remark Implicitly references pVCpu.
690	*/
691	# ifndef IEM_WITH_SETJMP
692	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
693	do \
694	{ \
695	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
696	if (rcStrict2 == VINF_SUCCESS) \
697	{ /* likely */ } \
698	else \
699	return rcStrict2; \
700	} while (0)
701	# else
702	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
703	# endif /* IEM_WITH_SETJMP */
704
705
706	# ifndef IEM_WITH_SETJMP
707	/**
708	* Fetches the next signed byte from the opcode stream.
709	*
710	* @returns Strict VBox status code.
711	* @param pVCpu The cross context virtual CPU structure of the calling thread.
712	* @param pi8 Where to return the signed byte.
713	*/
714	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
715	{
716	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
717	}
718	# endif /* !IEM_WITH_SETJMP */
719
720
721	/**
722	* Fetches the next signed byte from the opcode stream, returning automatically
723	* on failure.
724	*
725	* @param a_pi8 Where to return the signed byte.
726	* @remark Implicitly references pVCpu.
727	*/
728	# ifndef IEM_WITH_SETJMP
729	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
730	do \
731	{ \
732	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
733	if (rcStrict2 != VINF_SUCCESS) \
734	return rcStrict2; \
735	} while (0)
736	# else /* IEM_WITH_SETJMP */
737	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
738
739	# endif /* IEM_WITH_SETJMP */
740
741
742	# ifndef IEM_WITH_SETJMP
743	/**
744	* Fetches the next signed byte from the opcode stream, extending it to
745	* unsigned 16-bit.
746	*
747	* @returns Strict VBox status code.
748	* @param pVCpu The cross context virtual CPU structure of the calling thread.
749	* @param pu16 Where to return the unsigned word.
750	*/
751	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
752	{
753	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
754	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
755	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
756
757	*pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
758	pVCpu->iem.s.offOpcode = offOpcode + 1;
759	return VINF_SUCCESS;
760	}
761	# endif /* !IEM_WITH_SETJMP */
762
763	/**
764	* Fetches the next signed byte from the opcode stream and sign-extending it to
765	* a word, returning automatically on failure.
766	*
767	* @param a_pu16 Where to return the word.
768	* @remark Implicitly references pVCpu.
769	*/
770	# ifndef IEM_WITH_SETJMP
771	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
772	do \
773	{ \
774	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
775	if (rcStrict2 != VINF_SUCCESS) \
776	return rcStrict2; \
777	} while (0)
778	# else
779	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
780	# endif
781
782	# ifndef IEM_WITH_SETJMP
783	/**
784	* Fetches the next signed byte from the opcode stream, extending it to
785	* unsigned 32-bit.
786	*
787	* @returns Strict VBox status code.
788	* @param pVCpu The cross context virtual CPU structure of the calling thread.
789	* @param pu32 Where to return the unsigned dword.
790	*/
791	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
792	{
793	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
794	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
795	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
796
797	*pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
798	pVCpu->iem.s.offOpcode = offOpcode + 1;
799	return VINF_SUCCESS;
800	}
801	# endif /* !IEM_WITH_SETJMP */
802
803	/**
804	* Fetches the next signed byte from the opcode stream and sign-extending it to
805	* a word, returning automatically on failure.
806	*
807	* @param a_pu32 Where to return the word.
808	* @remark Implicitly references pVCpu.
809	*/
810	# ifndef IEM_WITH_SETJMP
811	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
812	do \
813	{ \
814	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
815	if (rcStrict2 != VINF_SUCCESS) \
816	return rcStrict2; \
817	} while (0)
818	# else
819	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
820	# endif
821
822
823	# ifndef IEM_WITH_SETJMP
824	/**
825	* Fetches the next signed byte from the opcode stream, extending it to
826	* unsigned 64-bit.
827	*
828	* @returns Strict VBox status code.
829	* @param pVCpu The cross context virtual CPU structure of the calling thread.
830	* @param pu64 Where to return the unsigned qword.
831	*/
832	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
833	{
834	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
835	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
836	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
837
838	*pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
839	pVCpu->iem.s.offOpcode = offOpcode + 1;
840	return VINF_SUCCESS;
841	}
842	# endif /* !IEM_WITH_SETJMP */
843
844	/**
845	* Fetches the next signed byte from the opcode stream and sign-extending it to
846	* a word, returning automatically on failure.
847	*
848	* @param a_pu64 Where to return the word.
849	* @remark Implicitly references pVCpu.
850	*/
851	# ifndef IEM_WITH_SETJMP
852	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
853	do \
854	{ \
855	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
856	if (rcStrict2 != VINF_SUCCESS) \
857	return rcStrict2; \
858	} while (0)
859	# else
860	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
861	# endif
862
863
864	# ifndef IEM_WITH_SETJMP
865	/**
866	* Fetches the next opcode byte.
867	*
868	* @returns Strict VBox status code.
869	* @param pVCpu The cross context virtual CPU structure of the
870	* calling thread.
871	* @param pu8 Where to return the opcode byte.
872	*/
873	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
874	{
875	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
876	pVCpu->iem.s.offModRm = offOpcode;
877	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
878	{
879	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
880	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
881	return VINF_SUCCESS;
882	}
883	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
884	}
885	# else /* IEM_WITH_SETJMP */
886	/**
887	* Fetches the next opcode byte, longjmp on error.
888	*
889	* @returns The opcode byte.
890	* @param pVCpu The cross context virtual CPU structure of the calling thread.
891	*/
892	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
893	{
894	# ifdef IEM_WITH_CODE_TLB
895	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
896	pVCpu->iem.s.offModRm = offBuf;
897	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
898	if (RT_LIKELY( pbBuf != NULL
899	&& offBuf < pVCpu->iem.s.cbInstrBuf))
900	{
901	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
902	return pbBuf[offBuf];
903	}
904	# else
905	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
906	pVCpu->iem.s.offModRm = offOpcode;
907	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
908	{
909	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
910	return pVCpu->iem.s.abOpcode[offOpcode];
911	}
912	# endif
913	return iemOpcodeGetNextU8SlowJmp(pVCpu);
914	}
915	# endif /* IEM_WITH_SETJMP */
916
917	/**
918	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
919	* on failure.
920	*
921	* Will note down the position of the ModR/M byte for VT-x exits.
922	*
923	* @param a_pbRm Where to return the RM opcode byte.
924	* @remark Implicitly references pVCpu.
925	*/
926	# ifndef IEM_WITH_SETJMP
927	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
928	do \
929	{ \
930	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
931	if (rcStrict2 == VINF_SUCCESS) \
932	{ /* likely */ } \
933	else \
934	return rcStrict2; \
935	} while (0)
936	# else
937	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
938	# endif /* IEM_WITH_SETJMP */
939
940
941	# ifndef IEM_WITH_SETJMP
942
943	/**
944	* Fetches the next opcode word.
945	*
946	* @returns Strict VBox status code.
947	* @param pVCpu The cross context virtual CPU structure of the calling thread.
948	* @param pu16 Where to return the opcode word.
949	*/
950	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
951	{
952	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
953	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
954	{
955	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
956	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
957	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
958	# else
959	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
960	# endif
961	return VINF_SUCCESS;
962	}
963	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
964	}
965
966	# else /* IEM_WITH_SETJMP */
967
968	/**
969	* Fetches the next opcode word, longjmp on error.
970	*
971	* @returns The opcode word.
972	* @param pVCpu The cross context virtual CPU structure of the calling thread.
973	*/
974	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
975	{
976	# ifdef IEM_WITH_CODE_TLB
977	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
978	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
979	if (RT_LIKELY( pbBuf != NULL
980	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
981	{
982	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
983	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
984	return (uint16_t const )&pbBuf[offBuf];
985	# else
986	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
987	# endif
988	}
989	# else /* !IEM_WITH_CODE_TLB */
990	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
991	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
992	{
993	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
994	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
995	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
996	# else
997	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
998	# endif
999	}
1000	# endif /* !IEM_WITH_CODE_TLB */
1001	return iemOpcodeGetNextU16SlowJmp(pVCpu);
1002	}
1003
1004	# endif /* IEM_WITH_SETJMP */
1005
1006	/**
1007	* Fetches the next opcode word, returns automatically on failure.
1008	*
1009	* @param a_pu16 Where to return the opcode word.
1010	* @remark Implicitly references pVCpu.
1011	*/
1012	# ifndef IEM_WITH_SETJMP
1013	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
1014	do \
1015	{ \
1016	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
1017	if (rcStrict2 != VINF_SUCCESS) \
1018	return rcStrict2; \
1019	} while (0)
1020	# else
1021	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
1022	# endif
1023
1024	# ifndef IEM_WITH_SETJMP
1025	/**
1026	* Fetches the next opcode word, zero extending it to a double word.
1027	*
1028	* @returns Strict VBox status code.
1029	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1030	* @param pu32 Where to return the opcode double word.
1031	*/
1032	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1033	{
1034	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1035	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1036	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1037
1038	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1039	pVCpu->iem.s.offOpcode = offOpcode + 2;
1040	return VINF_SUCCESS;
1041	}
1042	# endif /* !IEM_WITH_SETJMP */
1043
1044	/**
1045	* Fetches the next opcode word and zero extends it to a double word, returns
1046	* automatically on failure.
1047	*
1048	* @param a_pu32 Where to return the opcode double word.
1049	* @remark Implicitly references pVCpu.
1050	*/
1051	# ifndef IEM_WITH_SETJMP
1052	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1053	do \
1054	{ \
1055	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1056	if (rcStrict2 != VINF_SUCCESS) \
1057	return rcStrict2; \
1058	} while (0)
1059	# else
1060	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1061	# endif
1062
1063	# ifndef IEM_WITH_SETJMP
1064	/**
1065	* Fetches the next opcode word, zero extending it to a quad word.
1066	*
1067	* @returns Strict VBox status code.
1068	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1069	* @param pu64 Where to return the opcode quad word.
1070	*/
1071	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1072	{
1073	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1074	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1075	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1076
1077	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1078	pVCpu->iem.s.offOpcode = offOpcode + 2;
1079	return VINF_SUCCESS;
1080	}
1081	# endif /* !IEM_WITH_SETJMP */
1082
1083	/**
1084	* Fetches the next opcode word and zero extends it to a quad word, returns
1085	* automatically on failure.
1086	*
1087	* @param a_pu64 Where to return the opcode quad word.
1088	* @remark Implicitly references pVCpu.
1089	*/
1090	# ifndef IEM_WITH_SETJMP
1091	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1092	do \
1093	{ \
1094	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1095	if (rcStrict2 != VINF_SUCCESS) \
1096	return rcStrict2; \
1097	} while (0)
1098	# else
1099	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1100	# endif
1101
1102
1103	# ifndef IEM_WITH_SETJMP
1104	/**
1105	* Fetches the next signed word from the opcode stream.
1106	*
1107	* @returns Strict VBox status code.
1108	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1109	* @param pi16 Where to return the signed word.
1110	*/
1111	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1112	{
1113	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1114	}
1115	# endif /* !IEM_WITH_SETJMP */
1116
1117
1118	/**
1119	* Fetches the next signed word from the opcode stream, returning automatically
1120	* on failure.
1121	*
1122	* @param a_pi16 Where to return the signed word.
1123	* @remark Implicitly references pVCpu.
1124	*/
1125	# ifndef IEM_WITH_SETJMP
1126	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1127	do \
1128	{ \
1129	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1130	if (rcStrict2 != VINF_SUCCESS) \
1131	return rcStrict2; \
1132	} while (0)
1133	# else
1134	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1135	# endif
1136
1137	# ifndef IEM_WITH_SETJMP
1138
1139	/**
1140	* Fetches the next opcode dword.
1141	*
1142	* @returns Strict VBox status code.
1143	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1144	* @param pu32 Where to return the opcode double word.
1145	*/
1146	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1147	{
1148	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1149	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1150	{
1151	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1152	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1153	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1154	# else
1155	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1156	pVCpu->iem.s.abOpcode[offOpcode + 1],
1157	pVCpu->iem.s.abOpcode[offOpcode + 2],
1158	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1159	# endif
1160	return VINF_SUCCESS;
1161	}
1162	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1163	}
1164
1165	# else /* IEM_WITH_SETJMP */
1166
1167	/**
1168	* Fetches the next opcode dword, longjmp on error.
1169	*
1170	* @returns The opcode dword.
1171	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1172	*/
1173	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1174	{
1175	# ifdef IEM_WITH_CODE_TLB
1176	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1177	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1178	if (RT_LIKELY( pbBuf != NULL
1179	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1180	{
1181	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1182	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1183	return (uint32_t const )&pbBuf[offBuf];
1184	# else
1185	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1186	pbBuf[offBuf + 1],
1187	pbBuf[offBuf + 2],
1188	pbBuf[offBuf + 3]);
1189	# endif
1190	}
1191	# else
1192	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1193	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1194	{
1195	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1196	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1197	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1198	# else
1199	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1200	pVCpu->iem.s.abOpcode[offOpcode + 1],
1201	pVCpu->iem.s.abOpcode[offOpcode + 2],
1202	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1203	# endif
1204	}
1205	# endif
1206	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1207	}
1208
1209	# endif /* IEM_WITH_SETJMP */
1210
1211	/**
1212	* Fetches the next opcode dword, returns automatically on failure.
1213	*
1214	* @param a_pu32 Where to return the opcode dword.
1215	* @remark Implicitly references pVCpu.
1216	*/
1217	# ifndef IEM_WITH_SETJMP
1218	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1219	do \
1220	{ \
1221	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1222	if (rcStrict2 != VINF_SUCCESS) \
1223	return rcStrict2; \
1224	} while (0)
1225	# else
1226	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1227	# endif
1228
1229	# ifndef IEM_WITH_SETJMP
1230	/**
1231	* Fetches the next opcode dword, zero extending it to a quad word.
1232	*
1233	* @returns Strict VBox status code.
1234	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1235	* @param pu64 Where to return the opcode quad word.
1236	*/
1237	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1238	{
1239	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1240	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1241	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1242
1243	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1244	pVCpu->iem.s.abOpcode[offOpcode + 1],
1245	pVCpu->iem.s.abOpcode[offOpcode + 2],
1246	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1247	pVCpu->iem.s.offOpcode = offOpcode + 4;
1248	return VINF_SUCCESS;
1249	}
1250	# endif /* !IEM_WITH_SETJMP */
1251
1252	/**
1253	* Fetches the next opcode dword and zero extends it to a quad word, returns
1254	* automatically on failure.
1255	*
1256	* @param a_pu64 Where to return the opcode quad word.
1257	* @remark Implicitly references pVCpu.
1258	*/
1259	# ifndef IEM_WITH_SETJMP
1260	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1261	do \
1262	{ \
1263	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1264	if (rcStrict2 != VINF_SUCCESS) \
1265	return rcStrict2; \
1266	} while (0)
1267	# else
1268	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1269	# endif
1270
1271
1272	# ifndef IEM_WITH_SETJMP
1273	/**
1274	* Fetches the next signed double word from the opcode stream.
1275	*
1276	* @returns Strict VBox status code.
1277	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1278	* @param pi32 Where to return the signed double word.
1279	*/
1280	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1281	{
1282	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1283	}
1284	# endif
1285
1286	/**
1287	* Fetches the next signed double word from the opcode stream, returning
1288	* automatically on failure.
1289	*
1290	* @param a_pi32 Where to return the signed double word.
1291	* @remark Implicitly references pVCpu.
1292	*/
1293	# ifndef IEM_WITH_SETJMP
1294	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1295	do \
1296	{ \
1297	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1298	if (rcStrict2 != VINF_SUCCESS) \
1299	return rcStrict2; \
1300	} while (0)
1301	# else
1302	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1303	# endif
1304
1305	# ifndef IEM_WITH_SETJMP
1306	/**
1307	* Fetches the next opcode dword, sign extending it into a quad word.
1308	*
1309	* @returns Strict VBox status code.
1310	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1311	* @param pu64 Where to return the opcode quad word.
1312	*/
1313	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1314	{
1315	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1316	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1317	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1318
1319	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1320	pVCpu->iem.s.abOpcode[offOpcode + 1],
1321	pVCpu->iem.s.abOpcode[offOpcode + 2],
1322	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1323	*pu64 = (uint64_t)(int64_t)i32;
1324	pVCpu->iem.s.offOpcode = offOpcode + 4;
1325	return VINF_SUCCESS;
1326	}
1327	# endif /* !IEM_WITH_SETJMP */
1328
1329	/**
1330	* Fetches the next opcode double word and sign extends it to a quad word,
1331	* returns automatically on failure.
1332	*
1333	* @param a_pu64 Where to return the opcode quad word.
1334	* @remark Implicitly references pVCpu.
1335	*/
1336	# ifndef IEM_WITH_SETJMP
1337	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1338	do \
1339	{ \
1340	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1341	if (rcStrict2 != VINF_SUCCESS) \
1342	return rcStrict2; \
1343	} while (0)
1344	# else
1345	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1346	# endif
1347
1348	# ifndef IEM_WITH_SETJMP
1349
1350	/**
1351	* Fetches the next opcode qword.
1352	*
1353	* @returns Strict VBox status code.
1354	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1355	* @param pu64 Where to return the opcode qword.
1356	*/
1357	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1358	{
1359	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1360	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1361	{
1362	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1363	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1364	# else
1365	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1366	pVCpu->iem.s.abOpcode[offOpcode + 1],
1367	pVCpu->iem.s.abOpcode[offOpcode + 2],
1368	pVCpu->iem.s.abOpcode[offOpcode + 3],
1369	pVCpu->iem.s.abOpcode[offOpcode + 4],
1370	pVCpu->iem.s.abOpcode[offOpcode + 5],
1371	pVCpu->iem.s.abOpcode[offOpcode + 6],
1372	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1373	# endif
1374	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1375	return VINF_SUCCESS;
1376	}
1377	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1378	}
1379
1380	# else /* IEM_WITH_SETJMP */
1381
1382	/**
1383	* Fetches the next opcode qword, longjmp on error.
1384	*
1385	* @returns The opcode qword.
1386	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1387	*/
1388	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1389	{
1390	# ifdef IEM_WITH_CODE_TLB
1391	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1392	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1393	if (RT_LIKELY( pbBuf != NULL
1394	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1395	{
1396	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1397	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1398	return (uint64_t const )&pbBuf[offBuf];
1399	# else
1400	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1401	pbBuf[offBuf + 1],
1402	pbBuf[offBuf + 2],
1403	pbBuf[offBuf + 3],
1404	pbBuf[offBuf + 4],
1405	pbBuf[offBuf + 5],
1406	pbBuf[offBuf + 6],
1407	pbBuf[offBuf + 7]);
1408	# endif
1409	}
1410	# else
1411	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1412	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1413	{
1414	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1415	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1416	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1417	# else
1418	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1419	pVCpu->iem.s.abOpcode[offOpcode + 1],
1420	pVCpu->iem.s.abOpcode[offOpcode + 2],
1421	pVCpu->iem.s.abOpcode[offOpcode + 3],
1422	pVCpu->iem.s.abOpcode[offOpcode + 4],
1423	pVCpu->iem.s.abOpcode[offOpcode + 5],
1424	pVCpu->iem.s.abOpcode[offOpcode + 6],
1425	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1426	# endif
1427	}
1428	# endif
1429	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1430	}
1431
1432	# endif /* IEM_WITH_SETJMP */
1433
1434	/**
1435	* Fetches the next opcode quad word, returns automatically on failure.
1436	*
1437	* @param a_pu64 Where to return the opcode quad word.
1438	* @remark Implicitly references pVCpu.
1439	*/
1440	# ifndef IEM_WITH_SETJMP
1441	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1442	do \
1443	{ \
1444	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1445	if (rcStrict2 != VINF_SUCCESS) \
1446	return rcStrict2; \
1447	} while (0)
1448	# else
1449	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1450	# endif
1451
1452	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1453
1454
1455	/** @name Misc Worker Functions.
1456	* @{
1457	*/
1458
1459	/**
1460	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1461	* not (kind of obsolete now).
1462	*
1463	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1464	*/
1465	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1466
1467	/**
1468	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1469	*
1470	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1471	* @param a_fEfl The new EFLAGS.
1472	*/
1473	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1474
1475
1476	/**
1477	* Loads a NULL data selector into a selector register, both the hidden and
1478	* visible parts, in protected mode.
1479	*
1480	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1481	* @param pSReg Pointer to the segment register.
1482	* @param uRpl The RPL.
1483	*/
1484	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1485	{
1486	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1487	* data selector in protected mode. */
1488	pSReg->Sel = uRpl;
1489	pSReg->ValidSel = uRpl;
1490	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1491	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1492	{
1493	/* VT-x (Intel 3960x) observed doing something like this. */
1494	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1495	pSReg->u32Limit = UINT32_MAX;
1496	pSReg->u64Base = 0;
1497	}
1498	else
1499	{
1500	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1501	pSReg->u32Limit = 0;
1502	pSReg->u64Base = 0;
1503	}
1504	}
1505
1506	/** @} */
1507
1508
1509	/*
1510	*
1511	* Helpers routines.
1512	* Helpers routines.
1513	* Helpers routines.
1514	*
1515	*/
1516
1517	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1518
1519	/**
1520	* Recalculates the effective operand size.
1521	*
1522	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1523	*/
1524	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1525	{
1526	switch (IEM_GET_CPU_MODE(pVCpu))
1527	{
1528	case IEMMODE_16BIT:
1529	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1530	break;
1531	case IEMMODE_32BIT:
1532	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1533	break;
1534	case IEMMODE_64BIT:
1535	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1536	{
1537	case 0:
1538	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1539	break;
1540	case IEM_OP_PRF_SIZE_OP:
1541	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1542	break;
1543	case IEM_OP_PRF_SIZE_REX_W:
1544	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1545	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1546	break;
1547	}
1548	break;
1549	default:
1550	AssertFailed();
1551	}
1552	}
1553
1554
1555	/**
1556	* Sets the default operand size to 64-bit and recalculates the effective
1557	* operand size.
1558	*
1559	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1560	*/
1561	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1562	{
1563	Assert(IEM_IS_64BIT_CODE(pVCpu));
1564	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1565	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1566	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1567	else
1568	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1569	}
1570
1571
1572	/**
1573	* Sets the default operand size to 64-bit and recalculates the effective
1574	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1575	*
1576	* This is for the relative jumps.
1577	*
1578	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1579	*/
1580	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1581	{
1582	Assert(IEM_IS_64BIT_CODE(pVCpu));
1583	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1584	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1585	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1586	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1587	else
1588	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1589	}
1590
1591	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1592
1593
1594
1595	/** @name Register Access.
1596	* @{
1597	*/
1598
1599	/**
1600	* Gets a reference (pointer) to the specified hidden segment register.
1601	*
1602	* @returns Hidden register reference.
1603	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1604	* @param iSegReg The segment register.
1605	*/
1606	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1607	{
1608	Assert(iSegReg < X86_SREG_COUNT);
1609	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1610	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1611
1612	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1613	return pSReg;
1614	}
1615
1616
1617	/**
1618	* Ensures that the given hidden segment register is up to date.
1619	*
1620	* @returns Hidden register reference.
1621	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1622	* @param pSReg The segment register.
1623	*/
1624	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1625	{
1626	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1627	NOREF(pVCpu);
1628	return pSReg;
1629	}
1630
1631
1632	/**
1633	* Gets a reference (pointer) to the specified segment register (the selector
1634	* value).
1635	*
1636	* @returns Pointer to the selector variable.
1637	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1638	* @param iSegReg The segment register.
1639	*/
1640	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1641	{
1642	Assert(iSegReg < X86_SREG_COUNT);
1643	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1644	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1645	}
1646
1647
1648	/**
1649	* Fetches the selector value of a segment register.
1650	*
1651	* @returns The selector value.
1652	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1653	* @param iSegReg The segment register.
1654	*/
1655	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1656	{
1657	Assert(iSegReg < X86_SREG_COUNT);
1658	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1659	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1660	}
1661
1662
1663	/**
1664	* Fetches the base address value of a segment register.
1665	*
1666	* @returns The selector value.
1667	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1668	* @param iSegReg The segment register.
1669	*/
1670	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1671	{
1672	Assert(iSegReg < X86_SREG_COUNT);
1673	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1674	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1675	}
1676
1677
1678	/**
1679	* Gets a reference (pointer) to the specified general purpose register.
1680	*
1681	* @returns Register reference.
1682	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1683	* @param iReg The general purpose register.
1684	*/
1685	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1686	{
1687	Assert(iReg < 16);
1688	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1689	}
1690
1691
1692	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1693	/**
1694	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1695	*
1696	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1697	*
1698	* @returns Register reference.
1699	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1700	* @param iReg The register.
1701	*/
1702	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1703	{
1704	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1705	{
1706	Assert(iReg < 16);
1707	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1708	}
1709	/* high 8-bit register. */
1710	Assert(iReg < 8);
1711	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1712	}
1713	#endif
1714
1715
1716	/**
1717	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1718	* alternative version with extended (20) register index.
1719	*
1720	* @returns Register reference.
1721	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1722	* @param iRegEx The register. The 16 first are regular ones,
1723	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1724	*/
1725	DECLINLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1726	{
1727	if (iRegEx < 16)
1728	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1729
1730	/* high 8-bit register. */
1731	Assert(iRegEx < 20);
1732	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1733	}
1734
1735
1736	/**
1737	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1738	*
1739	* @returns Register reference.
1740	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1741	* @param iReg The register.
1742	*/
1743	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1744	{
1745	Assert(iReg < 16);
1746	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1747	}
1748
1749
1750	/**
1751	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1752	*
1753	* @returns Register reference.
1754	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1755	* @param iReg The register.
1756	*/
1757	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1758	{
1759	Assert(iReg < 16);
1760	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1761	}
1762
1763
1764	/**
1765	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1766	*
1767	* @returns Register reference.
1768	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1769	* @param iReg The register.
1770	*/
1771	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1772	{
1773	Assert(iReg < 16);
1774	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1775	}
1776
1777
1778	/**
1779	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1780	*
1781	* @returns Register reference.
1782	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1783	* @param iReg The register.
1784	*/
1785	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1786	{
1787	Assert(iReg < 64);
1788	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1789	}
1790
1791
1792	/**
1793	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1794	*
1795	* @returns Register reference.
1796	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1797	* @param iReg The register.
1798	*/
1799	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1800	{
1801	Assert(iReg < 16);
1802	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1803	}
1804
1805
1806	/**
1807	* Gets a reference (pointer) to the specified segment register's base address.
1808	*
1809	* @returns Segment register base address reference.
1810	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1811	* @param iSegReg The segment selector.
1812	*/
1813	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1814	{
1815	Assert(iSegReg < X86_SREG_COUNT);
1816	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1817	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1818	}
1819
1820
1821	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1822	/**
1823	* Fetches the value of a 8-bit general purpose register.
1824	*
1825	* @returns The register value.
1826	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1827	* @param iReg The register.
1828	*/
1829	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1830	{
1831	return *iemGRegRefU8(pVCpu, iReg);
1832	}
1833	#endif
1834
1835
1836	/**
1837	* Fetches the value of a 8-bit general purpose register, alternative version
1838	* with extended (20) register index.
1839
1840	* @returns The register value.
1841	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1842	* @param iRegEx The register. The 16 first are regular ones,
1843	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1844	*/
1845	DECLINLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1846	{
1847	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1848	}
1849
1850
1851	/**
1852	* Fetches the value of a 16-bit general purpose register.
1853	*
1854	* @returns The register value.
1855	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1856	* @param iReg The register.
1857	*/
1858	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1859	{
1860	Assert(iReg < 16);
1861	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1862	}
1863
1864
1865	/**
1866	* Fetches the value of a 32-bit general purpose register.
1867	*
1868	* @returns The register value.
1869	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1870	* @param iReg The register.
1871	*/
1872	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1873	{
1874	Assert(iReg < 16);
1875	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1876	}
1877
1878
1879	/**
1880	* Fetches the value of a 64-bit general purpose register.
1881	*
1882	* @returns The register value.
1883	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1884	* @param iReg The register.
1885	*/
1886	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1887	{
1888	Assert(iReg < 16);
1889	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1890	}
1891
1892
1893	/**
1894	* Get the address of the top of the stack.
1895	*
1896	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1897	*/
1898	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1899	{
1900	if (IEM_IS_64BIT_CODE(pVCpu))
1901	return pVCpu->cpum.GstCtx.rsp;
1902	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1903	return pVCpu->cpum.GstCtx.esp;
1904	return pVCpu->cpum.GstCtx.sp;
1905	}
1906
1907
1908	/**
1909	* Updates the RIP/EIP/IP to point to the next instruction.
1910	*
1911	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1912	* @param cbInstr The number of bytes to add.
1913	*/
1914	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1915	{
1916	/*
1917	* Advance RIP.
1918	*
1919	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1920	* while in all other modes except LM64 the updates are 32-bit. This means
1921	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1922	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1923	*
1924	* See PC wrap around tests in bs3-cpu-weird-1.
1925	*/
1926	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1927	uint64_t const uRipNext = uRipPrev + cbInstr;
1928	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1929	\|\| IEM_IS_64BIT_CODE(pVCpu)))
1930	pVCpu->cpum.GstCtx.rip = uRipNext;
1931	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1932	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1933	else
1934	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1935	}
1936
1937
1938	/**
1939	* Updates the EIP/IP to point to the next instruction - only for 32-bit and
1940	* 16-bit code.
1941	*
1942	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1943	* @param cbInstr The number of bytes to add.
1944	*/
1945	DECL_FORCE_INLINE(void) iemRegAddToEip32(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1946	{
1947	/* See comment in iemRegAddToRip. */
1948	uint32_t const uEipPrev = pVCpu->cpum.GstCtx.eip;
1949	uint32_t const uEipNext = uEipPrev + cbInstr;
1950	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1951	pVCpu->cpum.GstCtx.rip = (uint32_t)uEipNext;
1952	else
1953	pVCpu->cpum.GstCtx.rip = (uint16_t)uEipNext;
1954	}
1955
1956
1957	/**
1958	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1959	* following EFLAGS bits are set:
1960	* - X86_EFL_RF - clear it.
1961	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1962	* - X86_EFL_TF - generate single step \#DB trap.
1963	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1964	* instruction).
1965	*
1966	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1967	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
1968	* takes priority over both NMIs and hardware interrupts. So, neither is
1969	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1970	* either unsupported will be triggered on-top of any \#DB raised here.)
1971	*
1972	* The RF flag only needs to be cleared here as it only suppresses instruction
1973	* breakpoints which are not raised here (happens synchronously during
1974	* instruction fetching).
1975	*
1976	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
1977	* status has no bearing on whether \#DB exceptions are raised.
1978	*
1979	* @note This must NOT be called by the two instructions setting the
1980	* CPUMCTX_INHIBIT_SHADOW_SS flag.
1981	*
1982	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
1983	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
1984	* Stacks}
1985	*/
1986	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1987	{
1988	/*
1989	* Normally we're just here to clear RF and/or interrupt shadow bits.
1990	*/
1991	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
1992	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1993	else
1994	{
1995	/*
1996	* Raise a #DB or/and DBGF event.
1997	*/
1998	VBOXSTRICTRC rcStrict;
1999	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
2000	{
2001	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2002	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2003	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2004	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
2005	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2006	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2007	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2008	pVCpu->cpum.GstCtx.rflags.uBoth));
2009
2010	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
2011	rcStrict = iemRaiseDebugException(pVCpu);
2012
2013	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2014	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2015	{
2016	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2017	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2018	}
2019	}
2020	else
2021	{
2022	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2023	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2024	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2025	}
2026	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2027	return rcStrict;
2028	}
2029	return VINF_SUCCESS;
2030	}
2031
2032
2033	/**
2034	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2035	*
2036	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2037	*/
2038	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2039	{
2040	/*
2041	* We assume that most of the time nothing actually needs doing here.
2042	*/
2043	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2044	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2045	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
2046	return VINF_SUCCESS;
2047	return iemFinishInstructionWithFlagsSet(pVCpu);
2048	}
2049
2050
2051	/**
2052	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2053	* and CPUMCTX_INHIBIT_SHADOW.
2054	*
2055	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2056	* @param cbInstr The number of bytes to add.
2057	*/
2058	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2059	{
2060	iemRegAddToRip(pVCpu, cbInstr);
2061	return iemRegFinishClearingRF(pVCpu);
2062	}
2063
2064
2065	/**
2066	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
2067	* and CPUMCTX_INHIBIT_SHADOW.
2068	*
2069	* Only called from 64-code code.
2070	*
2071	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2072	* @param cbInstr The number of bytes to add.
2073	*/
2074	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2075	{
2076	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2077	return iemRegFinishClearingRF(pVCpu);
2078	}
2079
2080
2081	/**
2082	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2083	* CPUMCTX_INHIBIT_SHADOW.
2084	*
2085	* This is never from 64-code code.
2086	*
2087	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2088	* @param cbInstr The number of bytes to add.
2089	*/
2090	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2091	{
2092	iemRegAddToEip32(pVCpu, cbInstr);
2093	return iemRegFinishClearingRF(pVCpu);
2094	}
2095
2096
2097	/**
2098	* Extended version of iemFinishInstructionWithFlagsSet that goes with
2099	* iemRegAddToRipAndFinishingClearingRfEx.
2100	*
2101	* See iemFinishInstructionWithFlagsSet() for details.
2102	*/
2103	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2104	{
2105	/*
2106	* Raise a #DB.
2107	*/
2108	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2109	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2110	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
2111	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2112	/** @todo Do we set all pending \#DB events, or just one? */
2113	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2114	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2115	pVCpu->cpum.GstCtx.rflags.uBoth));
2116	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
2117	return iemRaiseDebugException(pVCpu);
2118	}
2119
2120
2121	/**
2122	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2123	* others potentially updating EFLAGS.TF.
2124	*
2125	* The single step event must be generated using the TF value at the start of
2126	* the instruction, not the new value set by it.
2127	*
2128	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2129	* @param cbInstr The number of bytes to add.
2130	* @param fEflOld The EFLAGS at the start of the instruction
2131	* execution.
2132	*/
2133	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2134	{
2135	iemRegAddToRip(pVCpu, cbInstr);
2136	if (!(fEflOld & X86_EFL_TF))
2137	return iemRegFinishClearingRF(pVCpu);
2138	return iemFinishInstructionWithTfSet(pVCpu);
2139	}
2140
2141
2142	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2143	/**
2144	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2145	*
2146	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2147	*/
2148	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2149	{
2150	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2151	}
2152	#endif
2153
2154
2155	/**
2156	* Adds to the stack pointer.
2157	*
2158	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2159	* @param cbToAdd The number of bytes to add (8-bit!).
2160	*/
2161	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2162	{
2163	if (IEM_IS_64BIT_CODE(pVCpu))
2164	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2165	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2166	pVCpu->cpum.GstCtx.esp += cbToAdd;
2167	else
2168	pVCpu->cpum.GstCtx.sp += cbToAdd;
2169	}
2170
2171
2172	/**
2173	* Subtracts from the stack pointer.
2174	*
2175	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2176	* @param cbToSub The number of bytes to subtract (8-bit!).
2177	*/
2178	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2179	{
2180	if (IEM_IS_64BIT_CODE(pVCpu))
2181	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2182	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2183	pVCpu->cpum.GstCtx.esp -= cbToSub;
2184	else
2185	pVCpu->cpum.GstCtx.sp -= cbToSub;
2186	}
2187
2188
2189	/**
2190	* Adds to the temporary stack pointer.
2191	*
2192	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2193	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2194	* @param cbToAdd The number of bytes to add (16-bit).
2195	*/
2196	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2197	{
2198	if (IEM_IS_64BIT_CODE(pVCpu))
2199	pTmpRsp->u += cbToAdd;
2200	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2201	pTmpRsp->DWords.dw0 += cbToAdd;
2202	else
2203	pTmpRsp->Words.w0 += cbToAdd;
2204	}
2205
2206
2207	/**
2208	* Subtracts from the temporary stack pointer.
2209	*
2210	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2211	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2212	* @param cbToSub The number of bytes to subtract.
2213	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2214	* expecting that.
2215	*/
2216	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2217	{
2218	if (IEM_IS_64BIT_CODE(pVCpu))
2219	pTmpRsp->u -= cbToSub;
2220	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2221	pTmpRsp->DWords.dw0 -= cbToSub;
2222	else
2223	pTmpRsp->Words.w0 -= cbToSub;
2224	}
2225
2226
2227	/**
2228	* Calculates the effective stack address for a push of the specified size as
2229	* well as the new RSP value (upper bits may be masked).
2230	*
2231	* @returns Effective stack addressf for the push.
2232	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2233	* @param cbItem The size of the stack item to pop.
2234	* @param puNewRsp Where to return the new RSP value.
2235	*/
2236	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2237	{
2238	RTUINT64U uTmpRsp;
2239	RTGCPTR GCPtrTop;
2240	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2241
2242	if (IEM_IS_64BIT_CODE(pVCpu))
2243	GCPtrTop = uTmpRsp.u -= cbItem;
2244	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2245	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2246	else
2247	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2248	*puNewRsp = uTmpRsp.u;
2249	return GCPtrTop;
2250	}
2251
2252
2253	/**
2254	* Gets the current stack pointer and calculates the value after a pop of the
2255	* specified size.
2256	*
2257	* @returns Current stack pointer.
2258	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2259	* @param cbItem The size of the stack item to pop.
2260	* @param puNewRsp Where to return the new RSP value.
2261	*/
2262	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2263	{
2264	RTUINT64U uTmpRsp;
2265	RTGCPTR GCPtrTop;
2266	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2267
2268	if (IEM_IS_64BIT_CODE(pVCpu))
2269	{
2270	GCPtrTop = uTmpRsp.u;
2271	uTmpRsp.u += cbItem;
2272	}
2273	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2274	{
2275	GCPtrTop = uTmpRsp.DWords.dw0;
2276	uTmpRsp.DWords.dw0 += cbItem;
2277	}
2278	else
2279	{
2280	GCPtrTop = uTmpRsp.Words.w0;
2281	uTmpRsp.Words.w0 += cbItem;
2282	}
2283	*puNewRsp = uTmpRsp.u;
2284	return GCPtrTop;
2285	}
2286
2287
2288	/**
2289	* Calculates the effective stack address for a push of the specified size as
2290	* well as the new temporary RSP value (upper bits may be masked).
2291	*
2292	* @returns Effective stack addressf for the push.
2293	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2294	* @param pTmpRsp The temporary stack pointer. This is updated.
2295	* @param cbItem The size of the stack item to pop.
2296	*/
2297	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2298	{
2299	RTGCPTR GCPtrTop;
2300
2301	if (IEM_IS_64BIT_CODE(pVCpu))
2302	GCPtrTop = pTmpRsp->u -= cbItem;
2303	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2304	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2305	else
2306	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2307	return GCPtrTop;
2308	}
2309
2310
2311	/**
2312	* Gets the effective stack address for a pop of the specified size and
2313	* calculates and updates the temporary RSP.
2314	*
2315	* @returns Current stack pointer.
2316	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2317	* @param pTmpRsp The temporary stack pointer. This is updated.
2318	* @param cbItem The size of the stack item to pop.
2319	*/
2320	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2321	{
2322	RTGCPTR GCPtrTop;
2323	if (IEM_IS_64BIT_CODE(pVCpu))
2324	{
2325	GCPtrTop = pTmpRsp->u;
2326	pTmpRsp->u += cbItem;
2327	}
2328	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2329	{
2330	GCPtrTop = pTmpRsp->DWords.dw0;
2331	pTmpRsp->DWords.dw0 += cbItem;
2332	}
2333	else
2334	{
2335	GCPtrTop = pTmpRsp->Words.w0;
2336	pTmpRsp->Words.w0 += cbItem;
2337	}
2338	return GCPtrTop;
2339	}
2340
2341	/** @} */
2342
2343
2344	/** @name FPU access and helpers.
2345	*
2346	* @{
2347	*/
2348
2349
2350	/**
2351	* Hook for preparing to use the host FPU.
2352	*
2353	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2354	*
2355	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2356	*/
2357	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2358	{
2359	#ifdef IN_RING3
2360	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2361	#else
2362	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2363	#endif
2364	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2365	}
2366
2367
2368	/**
2369	* Hook for preparing to use the host FPU for SSE.
2370	*
2371	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2372	*
2373	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2374	*/
2375	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2376	{
2377	iemFpuPrepareUsage(pVCpu);
2378	}
2379
2380
2381	/**
2382	* Hook for preparing to use the host FPU for AVX.
2383	*
2384	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2385	*
2386	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2387	*/
2388	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2389	{
2390	iemFpuPrepareUsage(pVCpu);
2391	}
2392
2393
2394	/**
2395	* Hook for actualizing the guest FPU state before the interpreter reads it.
2396	*
2397	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2398	*
2399	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2400	*/
2401	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2402	{
2403	#ifdef IN_RING3
2404	NOREF(pVCpu);
2405	#else
2406	CPUMRZFpuStateActualizeForRead(pVCpu);
2407	#endif
2408	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2409	}
2410
2411
2412	/**
2413	* Hook for actualizing the guest FPU state before the interpreter changes it.
2414	*
2415	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2416	*
2417	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2418	*/
2419	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2420	{
2421	#ifdef IN_RING3
2422	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2423	#else
2424	CPUMRZFpuStateActualizeForChange(pVCpu);
2425	#endif
2426	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2427	}
2428
2429
2430	/**
2431	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2432	* only.
2433	*
2434	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2435	*
2436	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2437	*/
2438	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2439	{
2440	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2441	NOREF(pVCpu);
2442	#else
2443	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2444	#endif
2445	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2446	}
2447
2448
2449	/**
2450	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2451	* read+write.
2452	*
2453	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2454	*
2455	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2456	*/
2457	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2458	{
2459	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2460	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2461	#else
2462	CPUMRZFpuStateActualizeForChange(pVCpu);
2463	#endif
2464	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2465
2466	/* Make sure any changes are loaded the next time around. */
2467	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2468	}
2469
2470
2471	/**
2472	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2473	* only.
2474	*
2475	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2476	*
2477	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2478	*/
2479	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2480	{
2481	#ifdef IN_RING3
2482	NOREF(pVCpu);
2483	#else
2484	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2485	#endif
2486	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2487	}
2488
2489
2490	/**
2491	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2492	* read+write.
2493	*
2494	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2495	*
2496	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2497	*/
2498	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2499	{
2500	#ifdef IN_RING3
2501	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2502	#else
2503	CPUMRZFpuStateActualizeForChange(pVCpu);
2504	#endif
2505	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2506
2507	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2508	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2509	}
2510
2511
2512	/**
2513	* Stores a QNaN value into a FPU register.
2514	*
2515	* @param pReg Pointer to the register.
2516	*/
2517	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2518	{
2519	pReg->au32[0] = UINT32_C(0x00000000);
2520	pReg->au32[1] = UINT32_C(0xc0000000);
2521	pReg->au16[4] = UINT16_C(0xffff);
2522	}
2523
2524
2525	/**
2526	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
2527	*
2528	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2529	* @param pFpuCtx The FPU context.
2530	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2531	*/
2532	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2533	{
2534	Assert(uFpuOpcode != UINT16_MAX);
2535	pFpuCtx->FOP = uFpuOpcode;
2536	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2537	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2538	{
2539	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2540	* happens in real mode here based on the fnsave and fnstenv images. */
2541	pFpuCtx->CS = 0;
2542	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2543	}
2544	else if (!IEM_IS_LONG_MODE(pVCpu))
2545	{
2546	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2547	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2548	}
2549	else
2550	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2551	}
2552
2553
2554	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2555	/**
2556	* Updates the FOP, FPU.CS and FPUIP registers.
2557	*
2558	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2559	* @param pFpuCtx The FPU context.
2560	*/
2561	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2562	{
2563	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2564	iemFpuUpdateOpcodeAndIpWorkerEx(pVCpu, pFpuCtx, pVCpu->iem.s.uFpuOpcode);
2565	}
2566	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
2567
2568
2569	/**
2570	* Marks the specified stack register as free (for FFREE).
2571	*
2572	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2573	* @param iStReg The register to free.
2574	*/
2575	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2576	{
2577	Assert(iStReg < 8);
2578	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2579	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2580	pFpuCtx->FTW &= ~RT_BIT(iReg);
2581	}
2582
2583
2584	/**
2585	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2586	*
2587	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2588	*/
2589	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2590	{
2591	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2592	uint16_t uFsw = pFpuCtx->FSW;
2593	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2594	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2595	uFsw &= ~X86_FSW_TOP_MASK;
2596	uFsw \|= uTop;
2597	pFpuCtx->FSW = uFsw;
2598	}
2599
2600
2601	/**
2602	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2603	*
2604	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2605	*/
2606	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2607	{
2608	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2609	uint16_t uFsw = pFpuCtx->FSW;
2610	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2611	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2612	uFsw &= ~X86_FSW_TOP_MASK;
2613	uFsw \|= uTop;
2614	pFpuCtx->FSW = uFsw;
2615	}
2616
2617
2618
2619
2620	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2621	{
2622	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2623	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2624	if (pFpuCtx->FTW & RT_BIT(iReg))
2625	return VINF_SUCCESS;
2626	return VERR_NOT_FOUND;
2627	}
2628
2629
2630	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2631	{
2632	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2633	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2634	if (pFpuCtx->FTW & RT_BIT(iReg))
2635	{
2636	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2637	return VINF_SUCCESS;
2638	}
2639	return VERR_NOT_FOUND;
2640	}
2641
2642
2643	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2644	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2645	{
2646	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2647	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2648	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2649	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2650	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2651	{
2652	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2653	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2654	return VINF_SUCCESS;
2655	}
2656	return VERR_NOT_FOUND;
2657	}
2658
2659
2660	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2661	{
2662	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2663	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2664	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2665	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2666	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2667	{
2668	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2669	return VINF_SUCCESS;
2670	}
2671	return VERR_NOT_FOUND;
2672	}
2673
2674
2675	/**
2676	* Rotates the stack registers when setting new TOS.
2677	*
2678	* @param pFpuCtx The FPU context.
2679	* @param iNewTop New TOS value.
2680	* @remarks We only do this to speed up fxsave/fxrstor which
2681	* arrange the FP registers in stack order.
2682	* MUST be done before writing the new TOS (FSW).
2683	*/
2684	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2685	{
2686	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2687	RTFLOAT80U ar80Temp[8];
2688
2689	if (iOldTop == iNewTop)
2690	return;
2691
2692	/* Unscrew the stack and get it into 'native' order. */
2693	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2694	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2695	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2696	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2697	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2698	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2699	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2700	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2701
2702	/* Now rotate the stack to the new position. */
2703	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2704	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2705	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2706	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2707	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2708	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2709	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2710	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2711	}
2712
2713
2714	/**
2715	* Updates the FPU exception status after FCW is changed.
2716	*
2717	* @param pFpuCtx The FPU context.
2718	*/
2719	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2720	{
2721	uint16_t u16Fsw = pFpuCtx->FSW;
2722	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2723	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2724	else
2725	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2726	pFpuCtx->FSW = u16Fsw;
2727	}
2728
2729
2730	/**
2731	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2732	*
2733	* @returns The full FTW.
2734	* @param pFpuCtx The FPU context.
2735	*/
2736	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
2737	{
2738	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2739	uint16_t u16Ftw = 0;
2740	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2741	for (unsigned iSt = 0; iSt < 8; iSt++)
2742	{
2743	unsigned const iReg = (iSt + iTop) & 7;
2744	if (!(u8Ftw & RT_BIT(iReg)))
2745	u16Ftw \|= 3 << (iReg * 2); /* empty */
2746	else
2747	{
2748	uint16_t uTag;
2749	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2750	if (pr80Reg->s.uExponent == 0x7fff)
2751	uTag = 2; /* Exponent is all 1's => Special. */
2752	else if (pr80Reg->s.uExponent == 0x0000)
2753	{
2754	if (pr80Reg->s.uMantissa == 0x0000)
2755	uTag = 1; /* All bits are zero => Zero. */
2756	else
2757	uTag = 2; /* Must be special. */
2758	}
2759	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2760	uTag = 0; /* Valid. */
2761	else
2762	uTag = 2; /* Must be special. */
2763
2764	u16Ftw \|= uTag << (iReg * 2);
2765	}
2766	}
2767
2768	return u16Ftw;
2769	}
2770
2771
2772	/**
2773	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2774	*
2775	* @returns The compressed FTW.
2776	* @param u16FullFtw The full FTW to convert.
2777	*/
2778	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
2779	{
2780	uint8_t u8Ftw = 0;
2781	for (unsigned i = 0; i < 8; i++)
2782	{
2783	if ((u16FullFtw & 3) != 3 /empty/)
2784	u8Ftw \|= RT_BIT(i);
2785	u16FullFtw >>= 2;
2786	}
2787
2788	return u8Ftw;
2789	}
2790
2791	/** @} */
2792
2793
2794	/** @name Memory access.
2795	*
2796	* @{
2797	*/
2798
2799
2800	/**
2801	* Checks whether alignment checks are enabled or not.
2802	*
2803	* @returns true if enabled, false if not.
2804	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2805	*/
2806	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
2807	{
2808	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2809	return IEM_GET_CPL(pVCpu) == 3
2810	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2811	}
2812
2813	/**
2814	* Checks if the given segment can be written to, raise the appropriate
2815	* exception if not.
2816	*
2817	* @returns VBox strict status code.
2818	*
2819	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2820	* @param pHid Pointer to the hidden register.
2821	* @param iSegReg The register number.
2822	* @param pu64BaseAddr Where to return the base address to use for the
2823	* segment. (In 64-bit code it may differ from the
2824	* base in the hidden segment.)
2825	*/
2826	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2827	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2828	{
2829	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2830
2831	if (IEM_IS_64BIT_CODE(pVCpu))
2832	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2833	else
2834	{
2835	if (!pHid->Attr.n.u1Present)
2836	{
2837	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2838	AssertRelease(uSel == 0);
2839	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2840	return iemRaiseGeneralProtectionFault0(pVCpu);
2841	}
2842
2843	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2844	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2845	&& !IEM_IS_64BIT_CODE(pVCpu) )
2846	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2847	*pu64BaseAddr = pHid->u64Base;
2848	}
2849	return VINF_SUCCESS;
2850	}
2851
2852
2853	/**
2854	* Checks if the given segment can be read from, raise the appropriate
2855	* exception if not.
2856	*
2857	* @returns VBox strict status code.
2858	*
2859	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2860	* @param pHid Pointer to the hidden register.
2861	* @param iSegReg The register number.
2862	* @param pu64BaseAddr Where to return the base address to use for the
2863	* segment. (In 64-bit code it may differ from the
2864	* base in the hidden segment.)
2865	*/
2866	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2867	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2868	{
2869	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2870
2871	if (IEM_IS_64BIT_CODE(pVCpu))
2872	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2873	else
2874	{
2875	if (!pHid->Attr.n.u1Present)
2876	{
2877	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2878	AssertRelease(uSel == 0);
2879	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2880	return iemRaiseGeneralProtectionFault0(pVCpu);
2881	}
2882
2883	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2884	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2885	*pu64BaseAddr = pHid->u64Base;
2886	}
2887	return VINF_SUCCESS;
2888	}
2889
2890
2891	/**
2892	* Maps a physical page.
2893	*
2894	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2895	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2896	* @param GCPhysMem The physical address.
2897	* @param fAccess The intended access.
2898	* @param ppvMem Where to return the mapping address.
2899	* @param pLock The PGM lock.
2900	*/
2901	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2902	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2903	{
2904	#ifdef IEM_LOG_MEMORY_WRITES
2905	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2906	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2907	#endif
2908
2909	/** @todo This API may require some improving later. A private deal with PGM
2910	* regarding locking and unlocking needs to be struct. A couple of TLBs
2911	* living in PGM, but with publicly accessible inlined access methods
2912	* could perhaps be an even better solution. */
2913	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2914	GCPhysMem,
2915	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2916	RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
2917	ppvMem,
2918	pLock);
2919	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2920	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2921
2922	return rc;
2923	}
2924
2925
2926	/**
2927	* Unmap a page previously mapped by iemMemPageMap.
2928	*
2929	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2930	* @param GCPhysMem The physical address.
2931	* @param fAccess The intended access.
2932	* @param pvMem What iemMemPageMap returned.
2933	* @param pLock The PGM lock.
2934	*/
2935	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2936	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2937	{
2938	NOREF(pVCpu);
2939	NOREF(GCPhysMem);
2940	NOREF(fAccess);
2941	NOREF(pvMem);
2942	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2943	}
2944
2945	#ifdef IEM_WITH_SETJMP
2946
2947	/** @todo slim this down */
2948	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
2949	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2950	{
2951	Assert(cbMem >= 1);
2952	Assert(iSegReg < X86_SREG_COUNT);
2953
2954	/*
2955	* 64-bit mode is simpler.
2956	*/
2957	if (IEM_IS_64BIT_CODE(pVCpu))
2958	{
2959	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2960	{
2961	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2962	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2963	GCPtrMem += pSel->u64Base;
2964	}
2965
2966	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2967	return GCPtrMem;
2968	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2969	}
2970	/*
2971	* 16-bit and 32-bit segmentation.
2972	*/
2973	else if (iSegReg != UINT8_MAX)
2974	{
2975	/** @todo Does this apply to segments with 4G-1 limit? */
2976	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2977	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2978	{
2979	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2980	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2981	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2982	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2983	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2984	\| X86_SEL_TYPE_CODE))
2985	{
2986	case X86DESCATTR_P: /* readonly data, expand up */
2987	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2988	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2989	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2990	/* expand up */
2991	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2992	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2993	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2994	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2995	break;
2996
2997	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2998	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2999	/* expand down */
3000	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3001	&& ( pSel->Attr.n.u1DefBig
3002	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
3003	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3004	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3005	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3006	break;
3007
3008	default:
3009	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3010	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3011	break;
3012	}
3013	}
3014	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3015	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3016	}
3017	/*
3018	* 32-bit flat address.
3019	*/
3020	else
3021	return GCPtrMem;
3022	}
3023
3024
3025	/** @todo slim this down */
3026	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3027	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3028	{
3029	Assert(cbMem >= 1);
3030	Assert(iSegReg < X86_SREG_COUNT);
3031
3032	/*
3033	* 64-bit mode is simpler.
3034	*/
3035	if (IEM_IS_64BIT_CODE(pVCpu))
3036	{
3037	if (iSegReg >= X86_SREG_FS)
3038	{
3039	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3040	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3041	GCPtrMem += pSel->u64Base;
3042	}
3043
3044	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3045	return GCPtrMem;
3046	}
3047	/*
3048	* 16-bit and 32-bit segmentation.
3049	*/
3050	else
3051	{
3052	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3053	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3054	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3055	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
3056	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
3057	{
3058	/* expand up */
3059	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
3060	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
3061	&& GCPtrLast32 > (uint32_t)GCPtrMem))
3062	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3063	}
3064	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
3065	{
3066	/* expand down */
3067	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
3068	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3069	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
3070	&& GCPtrLast32 > (uint32_t)GCPtrMem))
3071	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3072	}
3073	else
3074	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3075	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3076	}
3077	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3078	}
3079
3080	#endif /* IEM_WITH_SETJMP */
3081
3082	/**
3083	* Fakes a long mode stack selector for SS = 0.
3084	*
3085	* @param pDescSs Where to return the fake stack descriptor.
3086	* @param uDpl The DPL we want.
3087	*/
3088	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3089	{
3090	pDescSs->Long.au64[0] = 0;
3091	pDescSs->Long.au64[1] = 0;
3092	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3093	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3094	pDescSs->Long.Gen.u2Dpl = uDpl;
3095	pDescSs->Long.Gen.u1Present = 1;
3096	pDescSs->Long.Gen.u1Long = 1;
3097	}
3098
3099	/** @} */
3100
3101
3102	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3103
3104	/**
3105	* Gets CR0 fixed-0 bits in VMX operation.
3106	*
3107	* We do this rather than fetching what we report to the guest (in
3108	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3109	* values regardless of whether unrestricted-guest feature is available on the CPU.
3110	*
3111	* @returns CR0 fixed-0 bits.
3112	* @param pVCpu The cross context virtual CPU structure.
3113	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3114	* must be returned. When @c false, the CR0 fixed-0
3115	* bits for VMX root mode is returned.
3116	*
3117	*/
3118	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3119	{
3120	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3121
3122	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3123	if ( fVmxNonRootMode
3124	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3125	return VMX_V_CR0_FIXED0_UX;
3126	return VMX_V_CR0_FIXED0;
3127	}
3128
3129
3130	/**
3131	* Sets virtual-APIC write emulation as pending.
3132	*
3133	* @param pVCpu The cross context virtual CPU structure.
3134	* @param offApic The offset in the virtual-APIC page that was written.
3135	*/
3136	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3137	{
3138	Assert(offApic < XAPIC_OFF_END + 4);
3139
3140	/*
3141	* Record the currently updated APIC offset, as we need this later for figuring
3142	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
3143	* as for supplying the exit qualification when causing an APIC-write VM-exit.
3144	*/
3145	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3146
3147	/*
3148	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3149	* virtualization or APIC-write emulation).
3150	*/
3151	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3152	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3153	}
3154
3155	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3156
3157	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 100059

Download in other formats: