IEMInline.h@ 99298

Last change on this file since 99298 was 99298, checked in by vboxsync, 20 months ago
VMM/IEM: More work on processing MC blocks and generating threaded functions from them. bugref:10369
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 100.6 KB

Line
1	/* $Id: IEMInline.h 99298 2023-04-05 22:27:15Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu) RT_NOEXCEPT
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
185
186	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
187	/**
188	* Initializes the execution state.
189	*
190	* @param pVCpu The cross context virtual CPU structure of the
191	* calling thread.
192	* @param fBypassHandlers Whether to bypass access handlers.
193	*
194	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
195	* side-effects in strict builds.
196	*/
197	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers) RT_NOEXCEPT
198	{
199	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
200	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
207	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
208	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
209
210	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
211	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
212	# ifdef VBOX_STRICT
213	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
215	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
216	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
217	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
218	pVCpu->iem.s.uRexReg = 127;
219	pVCpu->iem.s.uRexB = 127;
220	pVCpu->iem.s.offModRm = 127;
221	pVCpu->iem.s.uRexIndex = 127;
222	pVCpu->iem.s.iEffSeg = 127;
223	pVCpu->iem.s.idxPrefix = 127;
224	pVCpu->iem.s.uVex3rdReg = 127;
225	pVCpu->iem.s.uVexLength = 127;
226	pVCpu->iem.s.fEvexStuff = 127;
227	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
228	# ifdef IEM_WITH_CODE_TLB
229	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
230	pVCpu->iem.s.pbInstrBuf = NULL;
231	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
232	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
233	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
234	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
235	# else
236	pVCpu->iem.s.offOpcode = 127;
237	pVCpu->iem.s.cbOpcode = 127;
238	# endif
239	# endif /* VBOX_STRICT */
240
241	pVCpu->iem.s.cActiveMappings = 0;
242	pVCpu->iem.s.iNextMapping = 0;
243	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
244	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
245	pVCpu->iem.s.fDisregardLock = false;
246	pVCpu->iem.s.fPendingInstructionBreakpoints = false;
247	pVCpu->iem.s.fPendingDataBreakpoints = false;
248	pVCpu->iem.s.fPendingIoBreakpoints = false;
249	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
250	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
251	{ /* likely */ }
252	else
253	iemInitPendingBreakpointsSlow(pVCpu);
254	}
255	# endif /* VBOX_INCLUDED_vmm_dbgf_h */
256
257
258	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
259	/**
260	* Performs a minimal reinitialization of the execution state.
261	*
262	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
263	* 'world-switch' types operations on the CPU. Currently only nested
264	* hardware-virtualization uses it.
265	*
266	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
267	*/
268	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
269	{
270	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
271	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
272
273	pVCpu->iem.s.uCpl = uCpl;
274	pVCpu->iem.s.enmCpuMode = enmMode;
275	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
276	pVCpu->iem.s.enmEffAddrMode = enmMode;
277	if (enmMode != IEMMODE_64BIT)
278	{
279	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
280	pVCpu->iem.s.enmEffOpSize = enmMode;
281	}
282	else
283	{
284	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
288	# ifndef IEM_WITH_CODE_TLB
289	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
290	pVCpu->iem.s.offOpcode = 0;
291	pVCpu->iem.s.cbOpcode = 0;
292	# endif
293	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
294	}
295	# endif
296
297
298	/**
299	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
300	*
301	* @param pVCpu The cross context virtual CPU structure of the
302	* calling thread.
303	*/
304	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
305	{
306	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
307	# ifdef VBOX_STRICT
308	# ifdef IEM_WITH_CODE_TLB
309	NOREF(pVCpu);
310	# else
311	pVCpu->iem.s.cbOpcode = 0;
312	# endif
313	# else
314	NOREF(pVCpu);
315	# endif
316	}
317
318
319	/**
320	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
321	*
322	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
323	*
324	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
325	* @param pVCpu The cross context virtual CPU structure of the calling thread.
326	* @param rcStrict The status code to fiddle.
327	*/
328	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
329	{
330	iemUninitExec(pVCpu);
331	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
332	}
333
334
335	/**
336	* Macro used by the IEMExec* method to check the given instruction length.
337	*
338	* Will return on failure!
339	*
340	* @param a_cbInstr The given instruction length.
341	* @param a_cbMin The minimum length.
342	*/
343	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
344	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
345	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
346
347
348	# ifndef IEM_WITH_SETJMP
349
350	/**
351	* Fetches the first opcode byte.
352	*
353	* @returns Strict VBox status code.
354	* @param pVCpu The cross context virtual CPU structure of the
355	* calling thread.
356	* @param pu8 Where to return the opcode byte.
357	*/
358	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
359	{
360	/*
361	* Check for hardware instruction breakpoints.
362	*/
363	if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints))
364	{ /* likely */ }
365	else
366	{
367	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
368	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
369	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
370	{ /* likely */ }
371	else if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
372	return iemRaiseDebugException(pVCpu);
373	else
374	return rcStrict;
375	}
376
377	/*
378	* Fetch the first opcode byte.
379	*/
380	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
381	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
382	{
383	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
384	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
385	return VINF_SUCCESS;
386	}
387	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
388	}
389
390	# else /* IEM_WITH_SETJMP */
391
392	/**
393	* Fetches the first opcode byte, longjmp on error.
394	*
395	* @returns The opcode byte.
396	* @param pVCpu The cross context virtual CPU structure of the calling thread.
397	*/
398	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
399	{
400	/*
401	* Check for hardware instruction breakpoints.
402	*/
403	if (RT_LIKELY(!pVCpu->iem.s.fPendingInstructionBreakpoints))
404	{ /* likely */ }
405	else
406	{
407	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
408	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
409	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
410	{ /* likely */ }
411	else
412	{
413	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
414	rcStrict = iemRaiseDebugException(pVCpu);
415	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
416	}
417	}
418
419	/*
420	* Fetch the first opcode byte.
421	*/
422	# ifdef IEM_WITH_CODE_TLB
423	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
424	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
425	if (RT_LIKELY( pbBuf != NULL
426	&& offBuf < pVCpu->iem.s.cbInstrBuf))
427	{
428	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
429	return pbBuf[offBuf];
430	}
431	# else
432	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
433	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
434	{
435	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
436	return pVCpu->iem.s.abOpcode[offOpcode];
437	}
438	# endif
439	return iemOpcodeGetNextU8SlowJmp(pVCpu);
440	}
441
442	# endif /* IEM_WITH_SETJMP */
443
444	/**
445	* Fetches the first opcode byte, returns/throws automatically on failure.
446	*
447	* @param a_pu8 Where to return the opcode byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	# ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
455	if (rcStrict2 == VINF_SUCCESS) \
456	{ /* likely */ } \
457	else \
458	return rcStrict2; \
459	} while (0)
460	# else
461	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
462	# endif /* IEM_WITH_SETJMP */
463
464
465	# ifndef IEM_WITH_SETJMP
466
467	/**
468	* Fetches the next opcode byte.
469	*
470	* @returns Strict VBox status code.
471	* @param pVCpu The cross context virtual CPU structure of the
472	* calling thread.
473	* @param pu8 Where to return the opcode byte.
474	*/
475	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
476	{
477	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
478	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
479	{
480	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
481	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
482	return VINF_SUCCESS;
483	}
484	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
485	}
486
487	# else /* IEM_WITH_SETJMP */
488
489	/**
490	* Fetches the next opcode byte, longjmp on error.
491	*
492	* @returns The opcode byte.
493	* @param pVCpu The cross context virtual CPU structure of the calling thread.
494	*/
495	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
496	{
497	# ifdef IEM_WITH_CODE_TLB
498	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
499	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
500	if (RT_LIKELY( pbBuf != NULL
501	&& offBuf < pVCpu->iem.s.cbInstrBuf))
502	{
503	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
504	return pbBuf[offBuf];
505	}
506	# else
507	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
508	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
509	{
510	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
511	return pVCpu->iem.s.abOpcode[offOpcode];
512	}
513	# endif
514	return iemOpcodeGetNextU8SlowJmp(pVCpu);
515	}
516
517	# endif /* IEM_WITH_SETJMP */
518
519	/**
520	* Fetches the next opcode byte, returns automatically on failure.
521	*
522	* @param a_pu8 Where to return the opcode byte.
523	* @remark Implicitly references pVCpu.
524	*/
525	# ifndef IEM_WITH_SETJMP
526	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
527	do \
528	{ \
529	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
530	if (rcStrict2 == VINF_SUCCESS) \
531	{ /* likely */ } \
532	else \
533	return rcStrict2; \
534	} while (0)
535	# else
536	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
537	# endif /* IEM_WITH_SETJMP */
538
539
540	# ifndef IEM_WITH_SETJMP
541	/**
542	* Fetches the next signed byte from the opcode stream.
543	*
544	* @returns Strict VBox status code.
545	* @param pVCpu The cross context virtual CPU structure of the calling thread.
546	* @param pi8 Where to return the signed byte.
547	*/
548	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
549	{
550	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
551	}
552	# endif /* !IEM_WITH_SETJMP */
553
554
555	/**
556	* Fetches the next signed byte from the opcode stream, returning automatically
557	* on failure.
558	*
559	* @param a_pi8 Where to return the signed byte.
560	* @remark Implicitly references pVCpu.
561	*/
562	# ifndef IEM_WITH_SETJMP
563	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
564	do \
565	{ \
566	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
567	if (rcStrict2 != VINF_SUCCESS) \
568	return rcStrict2; \
569	} while (0)
570	# else /* IEM_WITH_SETJMP */
571	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
572
573	# endif /* IEM_WITH_SETJMP */
574
575
576	# ifndef IEM_WITH_SETJMP
577	/**
578	* Fetches the next signed byte from the opcode stream, extending it to
579	* unsigned 16-bit.
580	*
581	* @returns Strict VBox status code.
582	* @param pVCpu The cross context virtual CPU structure of the calling thread.
583	* @param pu16 Where to return the unsigned word.
584	*/
585	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
586	{
587	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
588	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
589	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
590
591	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
592	pVCpu->iem.s.offOpcode = offOpcode + 1;
593	return VINF_SUCCESS;
594	}
595	# endif /* !IEM_WITH_SETJMP */
596
597	/**
598	* Fetches the next signed byte from the opcode stream and sign-extending it to
599	* a word, returning automatically on failure.
600	*
601	* @param a_pu16 Where to return the word.
602	* @remark Implicitly references pVCpu.
603	*/
604	# ifndef IEM_WITH_SETJMP
605	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
606	do \
607	{ \
608	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
609	if (rcStrict2 != VINF_SUCCESS) \
610	return rcStrict2; \
611	} while (0)
612	# else
613	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
614	# endif
615
616	# ifndef IEM_WITH_SETJMP
617	/**
618	* Fetches the next signed byte from the opcode stream, extending it to
619	* unsigned 32-bit.
620	*
621	* @returns Strict VBox status code.
622	* @param pVCpu The cross context virtual CPU structure of the calling thread.
623	* @param pu32 Where to return the unsigned dword.
624	*/
625	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
626	{
627	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
628	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
629	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
630
631	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
632	pVCpu->iem.s.offOpcode = offOpcode + 1;
633	return VINF_SUCCESS;
634	}
635	# endif /* !IEM_WITH_SETJMP */
636
637	/**
638	* Fetches the next signed byte from the opcode stream and sign-extending it to
639	* a word, returning automatically on failure.
640	*
641	* @param a_pu32 Where to return the word.
642	* @remark Implicitly references pVCpu.
643	*/
644	# ifndef IEM_WITH_SETJMP
645	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
646	do \
647	{ \
648	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
649	if (rcStrict2 != VINF_SUCCESS) \
650	return rcStrict2; \
651	} while (0)
652	# else
653	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
654	# endif
655
656
657	# ifndef IEM_WITH_SETJMP
658	/**
659	* Fetches the next signed byte from the opcode stream, extending it to
660	* unsigned 64-bit.
661	*
662	* @returns Strict VBox status code.
663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
664	* @param pu64 Where to return the unsigned qword.
665	*/
666	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
667	{
668	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
669	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
670	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
671
672	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
673	pVCpu->iem.s.offOpcode = offOpcode + 1;
674	return VINF_SUCCESS;
675	}
676	# endif /* !IEM_WITH_SETJMP */
677
678	/**
679	* Fetches the next signed byte from the opcode stream and sign-extending it to
680	* a word, returning automatically on failure.
681	*
682	* @param a_pu64 Where to return the word.
683	* @remark Implicitly references pVCpu.
684	*/
685	# ifndef IEM_WITH_SETJMP
686	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
687	do \
688	{ \
689	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
690	if (rcStrict2 != VINF_SUCCESS) \
691	return rcStrict2; \
692	} while (0)
693	# else
694	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
695	# endif
696
697
698	# ifndef IEM_WITH_SETJMP
699	/**
700	* Fetches the next opcode byte.
701	*
702	* @returns Strict VBox status code.
703	* @param pVCpu The cross context virtual CPU structure of the
704	* calling thread.
705	* @param pu8 Where to return the opcode byte.
706	*/
707	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
708	{
709	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
710	pVCpu->iem.s.offModRm = offOpcode;
711	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
712	{
713	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
714	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
715	return VINF_SUCCESS;
716	}
717	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
718	}
719	# else /* IEM_WITH_SETJMP */
720	/**
721	* Fetches the next opcode byte, longjmp on error.
722	*
723	* @returns The opcode byte.
724	* @param pVCpu The cross context virtual CPU structure of the calling thread.
725	*/
726	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
727	{
728	# ifdef IEM_WITH_CODE_TLB
729	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
730	pVCpu->iem.s.offModRm = offBuf;
731	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
732	if (RT_LIKELY( pbBuf != NULL
733	&& offBuf < pVCpu->iem.s.cbInstrBuf))
734	{
735	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
736	return pbBuf[offBuf];
737	}
738	# else
739	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
740	pVCpu->iem.s.offModRm = offOpcode;
741	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
742	{
743	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
744	return pVCpu->iem.s.abOpcode[offOpcode];
745	}
746	# endif
747	return iemOpcodeGetNextU8SlowJmp(pVCpu);
748	}
749	# endif /* IEM_WITH_SETJMP */
750
751	/**
752	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
753	* on failure.
754	*
755	* Will note down the position of the ModR/M byte for VT-x exits.
756	*
757	* @param a_pbRm Where to return the RM opcode byte.
758	* @remark Implicitly references pVCpu.
759	*/
760	# ifndef IEM_WITH_SETJMP
761	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
762	do \
763	{ \
764	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
765	if (rcStrict2 == VINF_SUCCESS) \
766	{ /* likely */ } \
767	else \
768	return rcStrict2; \
769	} while (0)
770	# else
771	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
772	# endif /* IEM_WITH_SETJMP */
773
774
775	# ifndef IEM_WITH_SETJMP
776
777	/**
778	* Fetches the next opcode word.
779	*
780	* @returns Strict VBox status code.
781	* @param pVCpu The cross context virtual CPU structure of the calling thread.
782	* @param pu16 Where to return the opcode word.
783	*/
784	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
785	{
786	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
787	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
788	{
789	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
790	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
791	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
792	# else
793	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
794	# endif
795	return VINF_SUCCESS;
796	}
797	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
798	}
799
800	# else /* IEM_WITH_SETJMP */
801
802	/**
803	* Fetches the next opcode word, longjmp on error.
804	*
805	* @returns The opcode word.
806	* @param pVCpu The cross context virtual CPU structure of the calling thread.
807	*/
808	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
809	{
810	# ifdef IEM_WITH_CODE_TLB
811	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
812	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
813	if (RT_LIKELY( pbBuf != NULL
814	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
815	{
816	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
817	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
818	return (uint16_t const )&pbBuf[offBuf];
819	# else
820	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
821	# endif
822	}
823	# else /* !IEM_WITH_CODE_TLB */
824	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
825	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
826	{
827	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
828	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
829	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
830	# else
831	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
832	# endif
833	}
834	# endif /* !IEM_WITH_CODE_TLB */
835	return iemOpcodeGetNextU16SlowJmp(pVCpu);
836	}
837
838	# endif /* IEM_WITH_SETJMP */
839
840	/**
841	* Fetches the next opcode word, returns automatically on failure.
842	*
843	* @param a_pu16 Where to return the opcode word.
844	* @remark Implicitly references pVCpu.
845	*/
846	# ifndef IEM_WITH_SETJMP
847	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
848	do \
849	{ \
850	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
851	if (rcStrict2 != VINF_SUCCESS) \
852	return rcStrict2; \
853	} while (0)
854	# else
855	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
856	# endif
857
858	# ifndef IEM_WITH_SETJMP
859	/**
860	* Fetches the next opcode word, zero extending it to a double word.
861	*
862	* @returns Strict VBox status code.
863	* @param pVCpu The cross context virtual CPU structure of the calling thread.
864	* @param pu32 Where to return the opcode double word.
865	*/
866	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
867	{
868	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
869	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
870	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
871
872	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
873	pVCpu->iem.s.offOpcode = offOpcode + 2;
874	return VINF_SUCCESS;
875	}
876	# endif /* !IEM_WITH_SETJMP */
877
878	/**
879	* Fetches the next opcode word and zero extends it to a double word, returns
880	* automatically on failure.
881	*
882	* @param a_pu32 Where to return the opcode double word.
883	* @remark Implicitly references pVCpu.
884	*/
885	# ifndef IEM_WITH_SETJMP
886	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
887	do \
888	{ \
889	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
890	if (rcStrict2 != VINF_SUCCESS) \
891	return rcStrict2; \
892	} while (0)
893	# else
894	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
895	# endif
896
897	# ifndef IEM_WITH_SETJMP
898	/**
899	* Fetches the next opcode word, zero extending it to a quad word.
900	*
901	* @returns Strict VBox status code.
902	* @param pVCpu The cross context virtual CPU structure of the calling thread.
903	* @param pu64 Where to return the opcode quad word.
904	*/
905	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
906	{
907	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
908	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
909	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
910
911	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
912	pVCpu->iem.s.offOpcode = offOpcode + 2;
913	return VINF_SUCCESS;
914	}
915	# endif /* !IEM_WITH_SETJMP */
916
917	/**
918	* Fetches the next opcode word and zero extends it to a quad word, returns
919	* automatically on failure.
920	*
921	* @param a_pu64 Where to return the opcode quad word.
922	* @remark Implicitly references pVCpu.
923	*/
924	# ifndef IEM_WITH_SETJMP
925	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
926	do \
927	{ \
928	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
929	if (rcStrict2 != VINF_SUCCESS) \
930	return rcStrict2; \
931	} while (0)
932	# else
933	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
934	# endif
935
936
937	# ifndef IEM_WITH_SETJMP
938	/**
939	* Fetches the next signed word from the opcode stream.
940	*
941	* @returns Strict VBox status code.
942	* @param pVCpu The cross context virtual CPU structure of the calling thread.
943	* @param pi16 Where to return the signed word.
944	*/
945	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
946	{
947	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
948	}
949	# endif /* !IEM_WITH_SETJMP */
950
951
952	/**
953	* Fetches the next signed word from the opcode stream, returning automatically
954	* on failure.
955	*
956	* @param a_pi16 Where to return the signed word.
957	* @remark Implicitly references pVCpu.
958	*/
959	# ifndef IEM_WITH_SETJMP
960	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
961	do \
962	{ \
963	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
964	if (rcStrict2 != VINF_SUCCESS) \
965	return rcStrict2; \
966	} while (0)
967	# else
968	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
969	# endif
970
971	# ifndef IEM_WITH_SETJMP
972
973	/**
974	* Fetches the next opcode dword.
975	*
976	* @returns Strict VBox status code.
977	* @param pVCpu The cross context virtual CPU structure of the calling thread.
978	* @param pu32 Where to return the opcode double word.
979	*/
980	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
981	{
982	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
983	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
984	{
985	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
986	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
987	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
988	# else
989	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
990	pVCpu->iem.s.abOpcode[offOpcode + 1],
991	pVCpu->iem.s.abOpcode[offOpcode + 2],
992	pVCpu->iem.s.abOpcode[offOpcode + 3]);
993	# endif
994	return VINF_SUCCESS;
995	}
996	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
997	}
998
999	# else /* IEM_WITH_SETJMP */
1000
1001	/**
1002	* Fetches the next opcode dword, longjmp on error.
1003	*
1004	* @returns The opcode dword.
1005	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1006	*/
1007	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1008	{
1009	# ifdef IEM_WITH_CODE_TLB
1010	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1011	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1012	if (RT_LIKELY( pbBuf != NULL
1013	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1014	{
1015	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1016	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1017	return (uint32_t const )&pbBuf[offBuf];
1018	# else
1019	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1020	pbBuf[offBuf + 1],
1021	pbBuf[offBuf + 2],
1022	pbBuf[offBuf + 3]);
1023	# endif
1024	}
1025	# else
1026	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1027	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1028	{
1029	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1030	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1031	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1032	# else
1033	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1034	pVCpu->iem.s.abOpcode[offOpcode + 1],
1035	pVCpu->iem.s.abOpcode[offOpcode + 2],
1036	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1037	# endif
1038	}
1039	# endif
1040	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1041	}
1042
1043	# endif /* IEM_WITH_SETJMP */
1044
1045	/**
1046	* Fetches the next opcode dword, returns automatically on failure.
1047	*
1048	* @param a_pu32 Where to return the opcode dword.
1049	* @remark Implicitly references pVCpu.
1050	*/
1051	# ifndef IEM_WITH_SETJMP
1052	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1053	do \
1054	{ \
1055	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1056	if (rcStrict2 != VINF_SUCCESS) \
1057	return rcStrict2; \
1058	} while (0)
1059	# else
1060	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1061	# endif
1062
1063	# ifndef IEM_WITH_SETJMP
1064	/**
1065	* Fetches the next opcode dword, zero extending it to a quad word.
1066	*
1067	* @returns Strict VBox status code.
1068	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1069	* @param pu64 Where to return the opcode quad word.
1070	*/
1071	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1072	{
1073	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1074	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1075	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1076
1077	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1078	pVCpu->iem.s.abOpcode[offOpcode + 1],
1079	pVCpu->iem.s.abOpcode[offOpcode + 2],
1080	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1081	pVCpu->iem.s.offOpcode = offOpcode + 4;
1082	return VINF_SUCCESS;
1083	}
1084	# endif /* !IEM_WITH_SETJMP */
1085
1086	/**
1087	* Fetches the next opcode dword and zero extends it to a quad word, returns
1088	* automatically on failure.
1089	*
1090	* @param a_pu64 Where to return the opcode quad word.
1091	* @remark Implicitly references pVCpu.
1092	*/
1093	# ifndef IEM_WITH_SETJMP
1094	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1095	do \
1096	{ \
1097	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1098	if (rcStrict2 != VINF_SUCCESS) \
1099	return rcStrict2; \
1100	} while (0)
1101	# else
1102	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1103	# endif
1104
1105
1106	# ifndef IEM_WITH_SETJMP
1107	/**
1108	* Fetches the next signed double word from the opcode stream.
1109	*
1110	* @returns Strict VBox status code.
1111	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1112	* @param pi32 Where to return the signed double word.
1113	*/
1114	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1115	{
1116	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1117	}
1118	# endif
1119
1120	/**
1121	* Fetches the next signed double word from the opcode stream, returning
1122	* automatically on failure.
1123	*
1124	* @param a_pi32 Where to return the signed double word.
1125	* @remark Implicitly references pVCpu.
1126	*/
1127	# ifndef IEM_WITH_SETJMP
1128	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1129	do \
1130	{ \
1131	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1132	if (rcStrict2 != VINF_SUCCESS) \
1133	return rcStrict2; \
1134	} while (0)
1135	# else
1136	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1137	# endif
1138
1139	# ifndef IEM_WITH_SETJMP
1140	/**
1141	* Fetches the next opcode dword, sign extending it into a quad word.
1142	*
1143	* @returns Strict VBox status code.
1144	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1145	* @param pu64 Where to return the opcode quad word.
1146	*/
1147	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1148	{
1149	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1150	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1151	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1152
1153	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1154	pVCpu->iem.s.abOpcode[offOpcode + 1],
1155	pVCpu->iem.s.abOpcode[offOpcode + 2],
1156	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1157	*pu64 = i32;
1158	pVCpu->iem.s.offOpcode = offOpcode + 4;
1159	return VINF_SUCCESS;
1160	}
1161	# endif /* !IEM_WITH_SETJMP */
1162
1163	/**
1164	* Fetches the next opcode double word and sign extends it to a quad word,
1165	* returns automatically on failure.
1166	*
1167	* @param a_pu64 Where to return the opcode quad word.
1168	* @remark Implicitly references pVCpu.
1169	*/
1170	# ifndef IEM_WITH_SETJMP
1171	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1172	do \
1173	{ \
1174	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1175	if (rcStrict2 != VINF_SUCCESS) \
1176	return rcStrict2; \
1177	} while (0)
1178	# else
1179	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1180	# endif
1181
1182	# ifndef IEM_WITH_SETJMP
1183
1184	/**
1185	* Fetches the next opcode qword.
1186	*
1187	* @returns Strict VBox status code.
1188	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1189	* @param pu64 Where to return the opcode qword.
1190	*/
1191	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1192	{
1193	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1194	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1195	{
1196	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1197	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1198	# else
1199	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1200	pVCpu->iem.s.abOpcode[offOpcode + 1],
1201	pVCpu->iem.s.abOpcode[offOpcode + 2],
1202	pVCpu->iem.s.abOpcode[offOpcode + 3],
1203	pVCpu->iem.s.abOpcode[offOpcode + 4],
1204	pVCpu->iem.s.abOpcode[offOpcode + 5],
1205	pVCpu->iem.s.abOpcode[offOpcode + 6],
1206	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1207	# endif
1208	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1209	return VINF_SUCCESS;
1210	}
1211	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1212	}
1213
1214	# else /* IEM_WITH_SETJMP */
1215
1216	/**
1217	* Fetches the next opcode qword, longjmp on error.
1218	*
1219	* @returns The opcode qword.
1220	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1221	*/
1222	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1223	{
1224	# ifdef IEM_WITH_CODE_TLB
1225	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1226	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1227	if (RT_LIKELY( pbBuf != NULL
1228	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1229	{
1230	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1231	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1232	return (uint64_t const )&pbBuf[offBuf];
1233	# else
1234	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1235	pbBuf[offBuf + 1],
1236	pbBuf[offBuf + 2],
1237	pbBuf[offBuf + 3],
1238	pbBuf[offBuf + 4],
1239	pbBuf[offBuf + 5],
1240	pbBuf[offBuf + 6],
1241	pbBuf[offBuf + 7]);
1242	# endif
1243	}
1244	# else
1245	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1246	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1247	{
1248	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1249	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1250	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1251	# else
1252	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1253	pVCpu->iem.s.abOpcode[offOpcode + 1],
1254	pVCpu->iem.s.abOpcode[offOpcode + 2],
1255	pVCpu->iem.s.abOpcode[offOpcode + 3],
1256	pVCpu->iem.s.abOpcode[offOpcode + 4],
1257	pVCpu->iem.s.abOpcode[offOpcode + 5],
1258	pVCpu->iem.s.abOpcode[offOpcode + 6],
1259	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1260	# endif
1261	}
1262	# endif
1263	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1264	}
1265
1266	# endif /* IEM_WITH_SETJMP */
1267
1268	/**
1269	* Fetches the next opcode quad word, returns automatically on failure.
1270	*
1271	* @param a_pu64 Where to return the opcode quad word.
1272	* @remark Implicitly references pVCpu.
1273	*/
1274	# ifndef IEM_WITH_SETJMP
1275	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1276	do \
1277	{ \
1278	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1279	if (rcStrict2 != VINF_SUCCESS) \
1280	return rcStrict2; \
1281	} while (0)
1282	# else
1283	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1284	# endif
1285
1286	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1287
1288
1289	/** @name Misc Worker Functions.
1290	* @{
1291	*/
1292
1293	/**
1294	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1295	* not (kind of obsolete now).
1296	*
1297	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1298	*/
1299	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1300
1301	/**
1302	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1303	*
1304	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1305	* @param a_fEfl The new EFLAGS.
1306	*/
1307	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1308
1309
1310	/**
1311	* Loads a NULL data selector into a selector register, both the hidden and
1312	* visible parts, in protected mode.
1313	*
1314	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1315	* @param pSReg Pointer to the segment register.
1316	* @param uRpl The RPL.
1317	*/
1318	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1319	{
1320	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1321	* data selector in protected mode. */
1322	pSReg->Sel = uRpl;
1323	pSReg->ValidSel = uRpl;
1324	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1325	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1326	{
1327	/* VT-x (Intel 3960x) observed doing something like this. */
1328	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1329	pSReg->u32Limit = UINT32_MAX;
1330	pSReg->u64Base = 0;
1331	}
1332	else
1333	{
1334	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1335	pSReg->u32Limit = 0;
1336	pSReg->u64Base = 0;
1337	}
1338	}
1339
1340	/** @} */
1341
1342
1343	/*
1344	*
1345	* Helpers routines.
1346	* Helpers routines.
1347	* Helpers routines.
1348	*
1349	*/
1350
1351	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1352
1353	/**
1354	* Recalculates the effective operand size.
1355	*
1356	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1357	*/
1358	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1359	{
1360	switch (pVCpu->iem.s.enmCpuMode)
1361	{
1362	case IEMMODE_16BIT:
1363	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1364	break;
1365	case IEMMODE_32BIT:
1366	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1367	break;
1368	case IEMMODE_64BIT:
1369	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1370	{
1371	case 0:
1372	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1373	break;
1374	case IEM_OP_PRF_SIZE_OP:
1375	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1376	break;
1377	case IEM_OP_PRF_SIZE_REX_W:
1378	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1379	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1380	break;
1381	}
1382	break;
1383	default:
1384	AssertFailed();
1385	}
1386	}
1387
1388
1389	/**
1390	* Sets the default operand size to 64-bit and recalculates the effective
1391	* operand size.
1392	*
1393	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1394	*/
1395	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1396	{
1397	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1398	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1399	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1400	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1401	else
1402	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1403	}
1404
1405
1406	/**
1407	* Sets the default operand size to 64-bit and recalculates the effective
1408	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1409	*
1410	* This is for the relative jumps.
1411	*
1412	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1413	*/
1414	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1415	{
1416	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1417	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1418	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1419	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1420	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1421	else
1422	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1423	}
1424
1425	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1426
1427
1428
1429	/** @name Register Access.
1430	* @{
1431	*/
1432
1433	/**
1434	* Gets a reference (pointer) to the specified hidden segment register.
1435	*
1436	* @returns Hidden register reference.
1437	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1438	* @param iSegReg The segment register.
1439	*/
1440	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1441	{
1442	Assert(iSegReg < X86_SREG_COUNT);
1443	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1444	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1445
1446	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1447	return pSReg;
1448	}
1449
1450
1451	/**
1452	* Ensures that the given hidden segment register is up to date.
1453	*
1454	* @returns Hidden register reference.
1455	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1456	* @param pSReg The segment register.
1457	*/
1458	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1459	{
1460	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1461	NOREF(pVCpu);
1462	return pSReg;
1463	}
1464
1465
1466	/**
1467	* Gets a reference (pointer) to the specified segment register (the selector
1468	* value).
1469	*
1470	* @returns Pointer to the selector variable.
1471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1472	* @param iSegReg The segment register.
1473	*/
1474	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1475	{
1476	Assert(iSegReg < X86_SREG_COUNT);
1477	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1478	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1479	}
1480
1481
1482	/**
1483	* Fetches the selector value of a segment register.
1484	*
1485	* @returns The selector value.
1486	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1487	* @param iSegReg The segment register.
1488	*/
1489	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1490	{
1491	Assert(iSegReg < X86_SREG_COUNT);
1492	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1493	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1494	}
1495
1496
1497	/**
1498	* Fetches the base address value of a segment register.
1499	*
1500	* @returns The selector value.
1501	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1502	* @param iSegReg The segment register.
1503	*/
1504	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1505	{
1506	Assert(iSegReg < X86_SREG_COUNT);
1507	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1508	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1509	}
1510
1511
1512	/**
1513	* Gets a reference (pointer) to the specified general purpose register.
1514	*
1515	* @returns Register reference.
1516	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1517	* @param iReg The general purpose register.
1518	*/
1519	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1520	{
1521	Assert(iReg < 16);
1522	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1523	}
1524
1525
1526	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1527	/**
1528	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1529	*
1530	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1531	*
1532	* @returns Register reference.
1533	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1534	* @param iReg The register.
1535	*/
1536	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1537	{
1538	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1539	{
1540	Assert(iReg < 16);
1541	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1542	}
1543	/* high 8-bit register. */
1544	Assert(iReg < 8);
1545	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1546	}
1547	#endif
1548
1549
1550	/**
1551	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1552	* alternative version with extended (20) register index.
1553	*
1554	* @returns Register reference.
1555	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1556	* @param iRegEx The register. The 16 first are regular ones,
1557	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1558	*/
1559	DECLINLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1560	{
1561	if (iRegEx < 16)
1562	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1563
1564	/* high 8-bit register. */
1565	Assert(iRegEx < 20);
1566	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1567	}
1568
1569
1570	/**
1571	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1572	*
1573	* @returns Register reference.
1574	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1575	* @param iReg The register.
1576	*/
1577	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1578	{
1579	Assert(iReg < 16);
1580	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1581	}
1582
1583
1584	/**
1585	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1586	*
1587	* @returns Register reference.
1588	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1589	* @param iReg The register.
1590	*/
1591	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1592	{
1593	Assert(iReg < 16);
1594	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1595	}
1596
1597
1598	/**
1599	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1600	*
1601	* @returns Register reference.
1602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1603	* @param iReg The register.
1604	*/
1605	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1606	{
1607	Assert(iReg < 16);
1608	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1609	}
1610
1611
1612	/**
1613	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1614	*
1615	* @returns Register reference.
1616	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1617	* @param iReg The register.
1618	*/
1619	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1620	{
1621	Assert(iReg < 64);
1622	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1623	}
1624
1625
1626	/**
1627	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1628	*
1629	* @returns Register reference.
1630	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1631	* @param iReg The register.
1632	*/
1633	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1634	{
1635	Assert(iReg < 16);
1636	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1637	}
1638
1639
1640	/**
1641	* Gets a reference (pointer) to the specified segment register's base address.
1642	*
1643	* @returns Segment register base address reference.
1644	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1645	* @param iSegReg The segment selector.
1646	*/
1647	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1648	{
1649	Assert(iSegReg < X86_SREG_COUNT);
1650	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1651	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1652	}
1653
1654
1655	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1656	/**
1657	* Fetches the value of a 8-bit general purpose register.
1658	*
1659	* @returns The register value.
1660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1661	* @param iReg The register.
1662	*/
1663	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1664	{
1665	return *iemGRegRefU8(pVCpu, iReg);
1666	}
1667	#endif
1668
1669
1670	/**
1671	* Fetches the value of a 8-bit general purpose register, alternative version
1672	* with extended (20) register index.
1673
1674	* @returns The register value.
1675	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1676	* @param iRegEx The register. The 16 first are regular ones,
1677	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1678	*/
1679	DECLINLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1680	{
1681	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1682	}
1683
1684
1685	/**
1686	* Fetches the value of a 16-bit general purpose register.
1687	*
1688	* @returns The register value.
1689	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1690	* @param iReg The register.
1691	*/
1692	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1693	{
1694	Assert(iReg < 16);
1695	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1696	}
1697
1698
1699	/**
1700	* Fetches the value of a 32-bit general purpose register.
1701	*
1702	* @returns The register value.
1703	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1704	* @param iReg The register.
1705	*/
1706	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1707	{
1708	Assert(iReg < 16);
1709	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1710	}
1711
1712
1713	/**
1714	* Fetches the value of a 64-bit general purpose register.
1715	*
1716	* @returns The register value.
1717	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1718	* @param iReg The register.
1719	*/
1720	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1721	{
1722	Assert(iReg < 16);
1723	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1724	}
1725
1726
1727	/**
1728	* Get the address of the top of the stack.
1729	*
1730	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1731	*/
1732	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1733	{
1734	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1735	return pVCpu->cpum.GstCtx.rsp;
1736	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1737	return pVCpu->cpum.GstCtx.esp;
1738	return pVCpu->cpum.GstCtx.sp;
1739	}
1740
1741
1742	/**
1743	* Updates the RIP/EIP/IP to point to the next instruction.
1744	*
1745	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1746	* @param cbInstr The number of bytes to add.
1747	*/
1748	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1749	{
1750	/*
1751	* Advance RIP.
1752	*
1753	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1754	* while in all other modes except LM64 the updates are 32-bit. This means
1755	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1756	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1757	*
1758	* See PC wrap around tests in bs3-cpu-weird-1.
1759	*/
1760	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1761	uint64_t const uRipNext = uRipPrev + cbInstr;
1762	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1763	\|\| pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT))
1764	pVCpu->cpum.GstCtx.rip = uRipNext;
1765	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1766	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1767	else
1768	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1769	}
1770
1771
1772	/**
1773	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1774	* following EFLAGS bits are set:
1775	* - X86_EFL_RF - clear it.
1776	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1777	* - X86_EFL_TF - generate single step \#DB trap.
1778	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1779	* instruction).
1780	*
1781	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1782	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
1783	* takes priority over both NMIs and hardware interrupts. So, neither is
1784	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1785	* either unsupported will be triggered on-top of any \#DB raised here.)
1786	*
1787	* The RF flag only needs to be cleared here as it only suppresses instruction
1788	* breakpoints which are not raised here (happens synchronously during
1789	* instruction fetching).
1790	*
1791	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
1792	* status has no bearing on whether \#DB exceptions are raised.
1793	*
1794	* @note This must NOT be called by the two instructions setting the
1795	* CPUMCTX_INHIBIT_SHADOW_SS flag.
1796	*
1797	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
1798	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
1799	* Stacks}
1800	*/
1801	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1802	{
1803	/*
1804	* Normally we're just here to clear RF and/or interrupt shadow bits.
1805	*/
1806	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
1807	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1808	else
1809	{
1810	/*
1811	* Raise a #DB or/and DBGF event.
1812	*/
1813	VBOXSTRICTRC rcStrict;
1814	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
1815	{
1816	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1817	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1818	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
1819	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
1820	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1821	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
1822	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1823	pVCpu->cpum.GstCtx.rflags.uBoth));
1824
1825	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1826	rcStrict = iemRaiseDebugException(pVCpu);
1827
1828	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
1829	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
1830	{
1831	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
1832	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
1833	}
1834	}
1835	else
1836	{
1837	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
1838	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
1839	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
1840	}
1841	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
1842	return rcStrict;
1843	}
1844	return VINF_SUCCESS;
1845	}
1846
1847
1848	/**
1849	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
1850	*
1851	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1852	*/
1853	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1854	{
1855	/*
1856	* We assume that most of the time nothing actually needs doing here.
1857	*/
1858	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1859	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
1860	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
1861	return VINF_SUCCESS;
1862	return iemFinishInstructionWithFlagsSet(pVCpu);
1863	}
1864
1865
1866	/**
1867	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
1868	* and CPUMCTX_INHIBIT_SHADOW.
1869	*
1870	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1871	* @param cbInstr The number of bytes to add.
1872	*/
1873	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
1874	{
1875	iemRegAddToRip(pVCpu, cbInstr);
1876	return iemRegFinishClearingRF(pVCpu);
1877	}
1878
1879
1880	/**
1881	* Extended version of iemFinishInstructionWithFlagsSet that goes with
1882	* iemRegAddToRipAndFinishingClearingRfEx.
1883	*
1884	* See iemFinishInstructionWithFlagsSet() for details.
1885	*/
1886	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1887	{
1888	/*
1889	* Raise a #DB.
1890	*/
1891	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1892	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1893	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
1894	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1895	/** @todo Do we set all pending \#DB events, or just one? */
1896	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
1897	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1898	pVCpu->cpum.GstCtx.rflags.uBoth));
1899	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
1900	return iemRaiseDebugException(pVCpu);
1901	}
1902
1903
1904	/**
1905	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
1906	* others potentially updating EFLAGS.TF.
1907	*
1908	* The single step event must be generated using the TF value at the start of
1909	* the instruction, not the new value set by it.
1910	*
1911	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1912	* @param cbInstr The number of bytes to add.
1913	* @param fEflOld The EFLAGS at the start of the instruction
1914	* execution.
1915	*/
1916	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
1917	{
1918	iemRegAddToRip(pVCpu, cbInstr);
1919	if (!(fEflOld & X86_EFL_TF))
1920	return iemRegFinishClearingRF(pVCpu);
1921	return iemFinishInstructionWithTfSet(pVCpu);
1922	}
1923
1924
1925	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1926	/**
1927	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1928	*
1929	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1930	*/
1931	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1932	{
1933	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1934	}
1935	#endif
1936
1937
1938	/**
1939	* Adds to the stack pointer.
1940	*
1941	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1942	* @param cbToAdd The number of bytes to add (8-bit!).
1943	*/
1944	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
1945	{
1946	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1947	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1948	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1949	pVCpu->cpum.GstCtx.esp += cbToAdd;
1950	else
1951	pVCpu->cpum.GstCtx.sp += cbToAdd;
1952	}
1953
1954
1955	/**
1956	* Subtracts from the stack pointer.
1957	*
1958	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1959	* @param cbToSub The number of bytes to subtract (8-bit!).
1960	*/
1961	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
1962	{
1963	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1964	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1965	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1966	pVCpu->cpum.GstCtx.esp -= cbToSub;
1967	else
1968	pVCpu->cpum.GstCtx.sp -= cbToSub;
1969	}
1970
1971
1972	/**
1973	* Adds to the temporary stack pointer.
1974	*
1975	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1976	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1977	* @param cbToAdd The number of bytes to add (16-bit).
1978	*/
1979	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
1980	{
1981	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1982	pTmpRsp->u += cbToAdd;
1983	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1984	pTmpRsp->DWords.dw0 += cbToAdd;
1985	else
1986	pTmpRsp->Words.w0 += cbToAdd;
1987	}
1988
1989
1990	/**
1991	* Subtracts from the temporary stack pointer.
1992	*
1993	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1994	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1995	* @param cbToSub The number of bytes to subtract.
1996	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1997	* expecting that.
1998	*/
1999	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2000	{
2001	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2002	pTmpRsp->u -= cbToSub;
2003	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2004	pTmpRsp->DWords.dw0 -= cbToSub;
2005	else
2006	pTmpRsp->Words.w0 -= cbToSub;
2007	}
2008
2009
2010	/**
2011	* Calculates the effective stack address for a push of the specified size as
2012	* well as the new RSP value (upper bits may be masked).
2013	*
2014	* @returns Effective stack addressf for the push.
2015	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2016	* @param cbItem The size of the stack item to pop.
2017	* @param puNewRsp Where to return the new RSP value.
2018	*/
2019	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2020	{
2021	RTUINT64U uTmpRsp;
2022	RTGCPTR GCPtrTop;
2023	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2024
2025	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2026	GCPtrTop = uTmpRsp.u -= cbItem;
2027	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2028	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2029	else
2030	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2031	*puNewRsp = uTmpRsp.u;
2032	return GCPtrTop;
2033	}
2034
2035
2036	/**
2037	* Gets the current stack pointer and calculates the value after a pop of the
2038	* specified size.
2039	*
2040	* @returns Current stack pointer.
2041	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2042	* @param cbItem The size of the stack item to pop.
2043	* @param puNewRsp Where to return the new RSP value.
2044	*/
2045	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2046	{
2047	RTUINT64U uTmpRsp;
2048	RTGCPTR GCPtrTop;
2049	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2050
2051	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2052	{
2053	GCPtrTop = uTmpRsp.u;
2054	uTmpRsp.u += cbItem;
2055	}
2056	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2057	{
2058	GCPtrTop = uTmpRsp.DWords.dw0;
2059	uTmpRsp.DWords.dw0 += cbItem;
2060	}
2061	else
2062	{
2063	GCPtrTop = uTmpRsp.Words.w0;
2064	uTmpRsp.Words.w0 += cbItem;
2065	}
2066	*puNewRsp = uTmpRsp.u;
2067	return GCPtrTop;
2068	}
2069
2070
2071	/**
2072	* Calculates the effective stack address for a push of the specified size as
2073	* well as the new temporary RSP value (upper bits may be masked).
2074	*
2075	* @returns Effective stack addressf for the push.
2076	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2077	* @param pTmpRsp The temporary stack pointer. This is updated.
2078	* @param cbItem The size of the stack item to pop.
2079	*/
2080	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2081	{
2082	RTGCPTR GCPtrTop;
2083
2084	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2085	GCPtrTop = pTmpRsp->u -= cbItem;
2086	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2087	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2088	else
2089	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2090	return GCPtrTop;
2091	}
2092
2093
2094	/**
2095	* Gets the effective stack address for a pop of the specified size and
2096	* calculates and updates the temporary RSP.
2097	*
2098	* @returns Current stack pointer.
2099	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2100	* @param pTmpRsp The temporary stack pointer. This is updated.
2101	* @param cbItem The size of the stack item to pop.
2102	*/
2103	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2104	{
2105	RTGCPTR GCPtrTop;
2106	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2107	{
2108	GCPtrTop = pTmpRsp->u;
2109	pTmpRsp->u += cbItem;
2110	}
2111	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2112	{
2113	GCPtrTop = pTmpRsp->DWords.dw0;
2114	pTmpRsp->DWords.dw0 += cbItem;
2115	}
2116	else
2117	{
2118	GCPtrTop = pTmpRsp->Words.w0;
2119	pTmpRsp->Words.w0 += cbItem;
2120	}
2121	return GCPtrTop;
2122	}
2123
2124	/** @} */
2125
2126
2127	/** @name FPU access and helpers.
2128	*
2129	* @{
2130	*/
2131
2132
2133	/**
2134	* Hook for preparing to use the host FPU.
2135	*
2136	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2137	*
2138	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2139	*/
2140	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2141	{
2142	#ifdef IN_RING3
2143	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2144	#else
2145	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2146	#endif
2147	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2148	}
2149
2150
2151	/**
2152	* Hook for preparing to use the host FPU for SSE.
2153	*
2154	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2155	*
2156	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2157	*/
2158	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2159	{
2160	iemFpuPrepareUsage(pVCpu);
2161	}
2162
2163
2164	/**
2165	* Hook for preparing to use the host FPU for AVX.
2166	*
2167	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2168	*
2169	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2170	*/
2171	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2172	{
2173	iemFpuPrepareUsage(pVCpu);
2174	}
2175
2176
2177	/**
2178	* Hook for actualizing the guest FPU state before the interpreter reads it.
2179	*
2180	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2181	*
2182	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2183	*/
2184	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2185	{
2186	#ifdef IN_RING3
2187	NOREF(pVCpu);
2188	#else
2189	CPUMRZFpuStateActualizeForRead(pVCpu);
2190	#endif
2191	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2192	}
2193
2194
2195	/**
2196	* Hook for actualizing the guest FPU state before the interpreter changes it.
2197	*
2198	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2199	*
2200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2201	*/
2202	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2203	{
2204	#ifdef IN_RING3
2205	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2206	#else
2207	CPUMRZFpuStateActualizeForChange(pVCpu);
2208	#endif
2209	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2210	}
2211
2212
2213	/**
2214	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2215	* only.
2216	*
2217	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2218	*
2219	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2220	*/
2221	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2222	{
2223	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2224	NOREF(pVCpu);
2225	#else
2226	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2227	#endif
2228	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2229	}
2230
2231
2232	/**
2233	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2234	* read+write.
2235	*
2236	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2237	*
2238	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2239	*/
2240	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2241	{
2242	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2243	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2244	#else
2245	CPUMRZFpuStateActualizeForChange(pVCpu);
2246	#endif
2247	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2248
2249	/* Make sure any changes are loaded the next time around. */
2250	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2251	}
2252
2253
2254	/**
2255	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2256	* only.
2257	*
2258	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2259	*
2260	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2261	*/
2262	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2263	{
2264	#ifdef IN_RING3
2265	NOREF(pVCpu);
2266	#else
2267	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2268	#endif
2269	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2270	}
2271
2272
2273	/**
2274	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2275	* read+write.
2276	*
2277	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2278	*
2279	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2280	*/
2281	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2282	{
2283	#ifdef IN_RING3
2284	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2285	#else
2286	CPUMRZFpuStateActualizeForChange(pVCpu);
2287	#endif
2288	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2289
2290	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2291	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2292	}
2293
2294
2295	/**
2296	* Stores a QNaN value into a FPU register.
2297	*
2298	* @param pReg Pointer to the register.
2299	*/
2300	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2301	{
2302	pReg->au32[0] = UINT32_C(0x00000000);
2303	pReg->au32[1] = UINT32_C(0xc0000000);
2304	pReg->au16[4] = UINT16_C(0xffff);
2305	}
2306
2307
2308	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2309	/**
2310	* Updates the FOP, FPU.CS and FPUIP registers.
2311	*
2312	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2313	* @param pFpuCtx The FPU context.
2314	*/
2315	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2316	{
2317	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2318	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
2319	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2320	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2321	{
2322	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2323	* happens in real mode here based on the fnsave and fnstenv images. */
2324	pFpuCtx->CS = 0;
2325	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2326	}
2327	else if (!IEM_IS_LONG_MODE(pVCpu))
2328	{
2329	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2330	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2331	}
2332	else
2333	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2334	}
2335	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
2336
2337
2338
2339
2340	/**
2341	* Marks the specified stack register as free (for FFREE).
2342	*
2343	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2344	* @param iStReg The register to free.
2345	*/
2346	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2347	{
2348	Assert(iStReg < 8);
2349	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2350	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2351	pFpuCtx->FTW &= ~RT_BIT(iReg);
2352	}
2353
2354
2355	/**
2356	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2357	*
2358	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2359	*/
2360	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2361	{
2362	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2363	uint16_t uFsw = pFpuCtx->FSW;
2364	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2365	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2366	uFsw &= ~X86_FSW_TOP_MASK;
2367	uFsw \|= uTop;
2368	pFpuCtx->FSW = uFsw;
2369	}
2370
2371
2372	/**
2373	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2374	*
2375	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2376	*/
2377	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2378	{
2379	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2380	uint16_t uFsw = pFpuCtx->FSW;
2381	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2382	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2383	uFsw &= ~X86_FSW_TOP_MASK;
2384	uFsw \|= uTop;
2385	pFpuCtx->FSW = uFsw;
2386	}
2387
2388
2389
2390
2391	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2392	{
2393	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2394	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2395	if (pFpuCtx->FTW & RT_BIT(iReg))
2396	return VINF_SUCCESS;
2397	return VERR_NOT_FOUND;
2398	}
2399
2400
2401	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2402	{
2403	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2404	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2405	if (pFpuCtx->FTW & RT_BIT(iReg))
2406	{
2407	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2408	return VINF_SUCCESS;
2409	}
2410	return VERR_NOT_FOUND;
2411	}
2412
2413
2414	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2415	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2416	{
2417	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2418	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2419	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2420	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2421	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2422	{
2423	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2424	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2425	return VINF_SUCCESS;
2426	}
2427	return VERR_NOT_FOUND;
2428	}
2429
2430
2431	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2432	{
2433	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2434	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2435	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2436	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2437	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2438	{
2439	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2440	return VINF_SUCCESS;
2441	}
2442	return VERR_NOT_FOUND;
2443	}
2444
2445
2446	/**
2447	* Rotates the stack registers when setting new TOS.
2448	*
2449	* @param pFpuCtx The FPU context.
2450	* @param iNewTop New TOS value.
2451	* @remarks We only do this to speed up fxsave/fxrstor which
2452	* arrange the FP registers in stack order.
2453	* MUST be done before writing the new TOS (FSW).
2454	*/
2455	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2456	{
2457	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2458	RTFLOAT80U ar80Temp[8];
2459
2460	if (iOldTop == iNewTop)
2461	return;
2462
2463	/* Unscrew the stack and get it into 'native' order. */
2464	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2465	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2466	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2467	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2468	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2469	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2470	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2471	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2472
2473	/* Now rotate the stack to the new position. */
2474	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2475	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2476	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2477	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2478	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2479	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2480	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2481	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2482	}
2483
2484
2485	/**
2486	* Updates the FPU exception status after FCW is changed.
2487	*
2488	* @param pFpuCtx The FPU context.
2489	*/
2490	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2491	{
2492	uint16_t u16Fsw = pFpuCtx->FSW;
2493	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2494	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2495	else
2496	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2497	pFpuCtx->FSW = u16Fsw;
2498	}
2499
2500
2501	/**
2502	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2503	*
2504	* @returns The full FTW.
2505	* @param pFpuCtx The FPU context.
2506	*/
2507	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
2508	{
2509	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2510	uint16_t u16Ftw = 0;
2511	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2512	for (unsigned iSt = 0; iSt < 8; iSt++)
2513	{
2514	unsigned const iReg = (iSt + iTop) & 7;
2515	if (!(u8Ftw & RT_BIT(iReg)))
2516	u16Ftw \|= 3 << (iReg * 2); /* empty */
2517	else
2518	{
2519	uint16_t uTag;
2520	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2521	if (pr80Reg->s.uExponent == 0x7fff)
2522	uTag = 2; /* Exponent is all 1's => Special. */
2523	else if (pr80Reg->s.uExponent == 0x0000)
2524	{
2525	if (pr80Reg->s.uMantissa == 0x0000)
2526	uTag = 1; /* All bits are zero => Zero. */
2527	else
2528	uTag = 2; /* Must be special. */
2529	}
2530	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2531	uTag = 0; /* Valid. */
2532	else
2533	uTag = 2; /* Must be special. */
2534
2535	u16Ftw \|= uTag << (iReg * 2);
2536	}
2537	}
2538
2539	return u16Ftw;
2540	}
2541
2542
2543	/**
2544	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2545	*
2546	* @returns The compressed FTW.
2547	* @param u16FullFtw The full FTW to convert.
2548	*/
2549	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
2550	{
2551	uint8_t u8Ftw = 0;
2552	for (unsigned i = 0; i < 8; i++)
2553	{
2554	if ((u16FullFtw & 3) != 3 /empty/)
2555	u8Ftw \|= RT_BIT(i);
2556	u16FullFtw >>= 2;
2557	}
2558
2559	return u8Ftw;
2560	}
2561
2562	/** @} */
2563
2564
2565	/** @name Memory access.
2566	*
2567	* @{
2568	*/
2569
2570
2571	/**
2572	* Checks whether alignment checks are enabled or not.
2573	*
2574	* @returns true if enabled, false if not.
2575	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2576	*/
2577	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
2578	{
2579	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2580	return pVCpu->iem.s.uCpl == 3
2581	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2582	}
2583
2584	/**
2585	* Checks if the given segment can be written to, raise the appropriate
2586	* exception if not.
2587	*
2588	* @returns VBox strict status code.
2589	*
2590	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2591	* @param pHid Pointer to the hidden register.
2592	* @param iSegReg The register number.
2593	* @param pu64BaseAddr Where to return the base address to use for the
2594	* segment. (In 64-bit code it may differ from the
2595	* base in the hidden segment.)
2596	*/
2597	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2598	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2599	{
2600	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2601
2602	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2603	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2604	else
2605	{
2606	if (!pHid->Attr.n.u1Present)
2607	{
2608	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2609	AssertRelease(uSel == 0);
2610	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2611	return iemRaiseGeneralProtectionFault0(pVCpu);
2612	}
2613
2614	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2615	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2616	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2617	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2618	*pu64BaseAddr = pHid->u64Base;
2619	}
2620	return VINF_SUCCESS;
2621	}
2622
2623
2624	/**
2625	* Checks if the given segment can be read from, raise the appropriate
2626	* exception if not.
2627	*
2628	* @returns VBox strict status code.
2629	*
2630	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2631	* @param pHid Pointer to the hidden register.
2632	* @param iSegReg The register number.
2633	* @param pu64BaseAddr Where to return the base address to use for the
2634	* segment. (In 64-bit code it may differ from the
2635	* base in the hidden segment.)
2636	*/
2637	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
2638	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
2639	{
2640	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2641
2642	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2643	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2644	else
2645	{
2646	if (!pHid->Attr.n.u1Present)
2647	{
2648	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2649	AssertRelease(uSel == 0);
2650	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2651	return iemRaiseGeneralProtectionFault0(pVCpu);
2652	}
2653
2654	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2655	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2656	*pu64BaseAddr = pHid->u64Base;
2657	}
2658	return VINF_SUCCESS;
2659	}
2660
2661
2662	/**
2663	* Maps a physical page.
2664	*
2665	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2666	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2667	* @param GCPhysMem The physical address.
2668	* @param fAccess The intended access.
2669	* @param ppvMem Where to return the mapping address.
2670	* @param pLock The PGM lock.
2671	*/
2672	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2673	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2674	{
2675	#ifdef IEM_LOG_MEMORY_WRITES
2676	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2677	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2678	#endif
2679
2680	/** @todo This API may require some improving later. A private deal with PGM
2681	* regarding locking and unlocking needs to be struct. A couple of TLBs
2682	* living in PGM, but with publicly accessible inlined access methods
2683	* could perhaps be an even better solution. */
2684	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2685	GCPhysMem,
2686	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2687	pVCpu->iem.s.fBypassHandlers,
2688	ppvMem,
2689	pLock);
2690	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2691	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2692
2693	return rc;
2694	}
2695
2696
2697	/**
2698	* Unmap a page previously mapped by iemMemPageMap.
2699	*
2700	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2701	* @param GCPhysMem The physical address.
2702	* @param fAccess The intended access.
2703	* @param pvMem What iemMemPageMap returned.
2704	* @param pLock The PGM lock.
2705	*/
2706	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
2707	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
2708	{
2709	NOREF(pVCpu);
2710	NOREF(GCPhysMem);
2711	NOREF(fAccess);
2712	NOREF(pvMem);
2713	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2714	}
2715
2716	#ifdef IEM_WITH_SETJMP
2717
2718	/** @todo slim this down */
2719	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
2720	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2721	{
2722	Assert(cbMem >= 1);
2723	Assert(iSegReg < X86_SREG_COUNT);
2724
2725	/*
2726	* 64-bit mode is simpler.
2727	*/
2728	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2729	{
2730	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2731	{
2732	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2733	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2734	GCPtrMem += pSel->u64Base;
2735	}
2736
2737	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2738	return GCPtrMem;
2739	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2740	}
2741	/*
2742	* 16-bit and 32-bit segmentation.
2743	*/
2744	else if (iSegReg != UINT8_MAX)
2745	{
2746	/** @todo Does this apply to segments with 4G-1 limit? */
2747	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2748	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2749	{
2750	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2751	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2752	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2753	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2754	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2755	\| X86_SEL_TYPE_CODE))
2756	{
2757	case X86DESCATTR_P: /* readonly data, expand up */
2758	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2759	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2760	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2761	/* expand up */
2762	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2763	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2764	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2765	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2766	break;
2767
2768	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2769	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2770	/* expand down */
2771	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2772	&& ( pSel->Attr.n.u1DefBig
2773	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2774	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2775	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2776	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2777	break;
2778
2779	default:
2780	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2781	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2782	break;
2783	}
2784	}
2785	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2786	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2787	}
2788	/*
2789	* 32-bit flat address.
2790	*/
2791	else
2792	return GCPtrMem;
2793	}
2794
2795
2796	/** @todo slim this down */
2797	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
2798	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
2799	{
2800	Assert(cbMem >= 1);
2801	Assert(iSegReg < X86_SREG_COUNT);
2802
2803	/*
2804	* 64-bit mode is simpler.
2805	*/
2806	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2807	{
2808	if (iSegReg >= X86_SREG_FS)
2809	{
2810	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2811	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2812	GCPtrMem += pSel->u64Base;
2813	}
2814
2815	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2816	return GCPtrMem;
2817	}
2818	/*
2819	* 16-bit and 32-bit segmentation.
2820	*/
2821	else
2822	{
2823	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2824	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2825	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2826	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2827	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2828	{
2829	/* expand up */
2830	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2831	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2832	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2833	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2834	}
2835	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2836	{
2837	/* expand down */
2838	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2839	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2840	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2841	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2842	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2843	}
2844	else
2845	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2846	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2847	}
2848	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2849	}
2850
2851	#endif /* IEM_WITH_SETJMP */
2852
2853	/**
2854	* Fakes a long mode stack selector for SS = 0.
2855	*
2856	* @param pDescSs Where to return the fake stack descriptor.
2857	* @param uDpl The DPL we want.
2858	*/
2859	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
2860	{
2861	pDescSs->Long.au64[0] = 0;
2862	pDescSs->Long.au64[1] = 0;
2863	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2864	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2865	pDescSs->Long.Gen.u2Dpl = uDpl;
2866	pDescSs->Long.Gen.u1Present = 1;
2867	pDescSs->Long.Gen.u1Long = 1;
2868	}
2869
2870	/** @} */
2871
2872
2873	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2874
2875	/**
2876	* Gets CR0 fixed-0 bits in VMX operation.
2877	*
2878	* We do this rather than fetching what we report to the guest (in
2879	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2880	* values regardless of whether unrestricted-guest feature is available on the CPU.
2881	*
2882	* @returns CR0 fixed-0 bits.
2883	* @param pVCpu The cross context virtual CPU structure.
2884	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
2885	* must be returned. When @c false, the CR0 fixed-0
2886	* bits for VMX root mode is returned.
2887	*
2888	*/
2889	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
2890	{
2891	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2892
2893	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
2894	if ( fVmxNonRootMode
2895	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
2896	return VMX_V_CR0_FIXED0_UX;
2897	return VMX_V_CR0_FIXED0;
2898	}
2899
2900
2901	/**
2902	* Sets virtual-APIC write emulation as pending.
2903	*
2904	* @param pVCpu The cross context virtual CPU structure.
2905	* @param offApic The offset in the virtual-APIC page that was written.
2906	*/
2907	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
2908	{
2909	Assert(offApic < XAPIC_OFF_END + 4);
2910
2911	/*
2912	* Record the currently updated APIC offset, as we need this later for figuring
2913	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2914	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2915	*/
2916	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2917
2918	/*
2919	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2920	* virtualization or APIC-write emulation).
2921	*/
2922	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2923	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2924	}
2925
2926	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2927
2928	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 99298

Download in other formats: