IEMInline.h@ 97334

Last change on this file since 97334 was 97334, checked in by vboxsync, 2 years ago
VMM/IEM: Address issues in iemRegAddToRipAndClearRF wrt wrap-around.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 88.0 KB

Line
1	/* $Id: IEMInline.h 97334 2022-10-28 14:17:25Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184
185	/**
186	* Initializes the execution state.
187	*
188	* @param pVCpu The cross context virtual CPU structure of the
189	* calling thread.
190	* @param fBypassHandlers Whether to bypass access handlers.
191	*
192	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
193	* side-effects in strict builds.
194	*/
195	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
196	{
197	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
198	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
199	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
200	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
207
208	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
209	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
210	#ifdef VBOX_STRICT
211	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
212	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
213	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
215	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
216	pVCpu->iem.s.uRexReg = 127;
217	pVCpu->iem.s.uRexB = 127;
218	pVCpu->iem.s.offModRm = 127;
219	pVCpu->iem.s.uRexIndex = 127;
220	pVCpu->iem.s.iEffSeg = 127;
221	pVCpu->iem.s.idxPrefix = 127;
222	pVCpu->iem.s.uVex3rdReg = 127;
223	pVCpu->iem.s.uVexLength = 127;
224	pVCpu->iem.s.fEvexStuff = 127;
225	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
226	# ifdef IEM_WITH_CODE_TLB
227	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
228	pVCpu->iem.s.pbInstrBuf = NULL;
229	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
230	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
231	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
232	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
233	# else
234	pVCpu->iem.s.offOpcode = 127;
235	pVCpu->iem.s.cbOpcode = 127;
236	# endif
237	#endif
238
239	pVCpu->iem.s.cActiveMappings = 0;
240	pVCpu->iem.s.iNextMapping = 0;
241	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
242	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
243	#if 0
244	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
245	if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
246	&& CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
247	{
248	PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
249	Assert(pVmcs);
250	RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
251	if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
252	{
253	int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
254	pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
255	NIL_RTR0PTR /* pvUserR0 /, NIL_RTRCPTR / pvUserRC /, NULL / pszDesc */);
256	AssertRC(rc);
257	}
258	}
259	#endif
260	#endif
261	}
262
263	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
264	/**
265	* Performs a minimal reinitialization of the execution state.
266	*
267	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
268	* 'world-switch' types operations on the CPU. Currently only nested
269	* hardware-virtualization uses it.
270	*
271	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
272	*/
273	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu)
274	{
275	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
276	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
277
278	pVCpu->iem.s.uCpl = uCpl;
279	pVCpu->iem.s.enmCpuMode = enmMode;
280	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
281	pVCpu->iem.s.enmEffAddrMode = enmMode;
282	if (enmMode != IEMMODE_64BIT)
283	{
284	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	else
288	{
289	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
290	pVCpu->iem.s.enmEffOpSize = enmMode;
291	}
292	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
293	#ifndef IEM_WITH_CODE_TLB
294	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
295	pVCpu->iem.s.offOpcode = 0;
296	pVCpu->iem.s.cbOpcode = 0;
297	#endif
298	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
299	}
300	#endif
301
302	/**
303	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
304	*
305	* @param pVCpu The cross context virtual CPU structure of the
306	* calling thread.
307	*/
308	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
309	{
310	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
311	#ifdef VBOX_STRICT
312	# ifdef IEM_WITH_CODE_TLB
313	NOREF(pVCpu);
314	# else
315	pVCpu->iem.s.cbOpcode = 0;
316	# endif
317	#else
318	NOREF(pVCpu);
319	#endif
320	}
321
322
323	/**
324	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
325	*
326	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
327	*
328	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
329	* @param pVCpu The cross context virtual CPU structure of the calling thread.
330	* @param rcStrict The status code to fiddle.
331	*/
332	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
333	{
334	iemUninitExec(pVCpu);
335	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
336	}
337
338
339	/**
340	* Macro used by the IEMExec* method to check the given instruction length.
341	*
342	* Will return on failure!
343	*
344	* @param a_cbInstr The given instruction length.
345	* @param a_cbMin The minimum length.
346	*/
347	#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
348	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
349	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
350
351
352
353	#ifndef IEM_WITH_SETJMP
354
355	/**
356	* Fetches the next opcode byte.
357	*
358	* @returns Strict VBox status code.
359	* @param pVCpu The cross context virtual CPU structure of the
360	* calling thread.
361	* @param pu8 Where to return the opcode byte.
362	*/
363	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
364	{
365	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
366	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
367	{
368	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
369	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
370	return VINF_SUCCESS;
371	}
372	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
373	}
374
375	#else /* IEM_WITH_SETJMP */
376
377	/**
378	* Fetches the next opcode byte, longjmp on error.
379	*
380	* @returns The opcode byte.
381	* @param pVCpu The cross context virtual CPU structure of the calling thread.
382	*/
383	DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
384	{
385	# ifdef IEM_WITH_CODE_TLB
386	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
387	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
388	if (RT_LIKELY( pbBuf != NULL
389	&& offBuf < pVCpu->iem.s.cbInstrBuf))
390	{
391	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
392	return pbBuf[offBuf];
393	}
394	# else
395	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
396	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
397	{
398	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
399	return pVCpu->iem.s.abOpcode[offOpcode];
400	}
401	# endif
402	return iemOpcodeGetNextU8SlowJmp(pVCpu);
403	}
404
405	#endif /* IEM_WITH_SETJMP */
406
407	/**
408	* Fetches the next opcode byte, returns automatically on failure.
409	*
410	* @param a_pu8 Where to return the opcode byte.
411	* @remark Implicitly references pVCpu.
412	*/
413	#ifndef IEM_WITH_SETJMP
414	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
415	do \
416	{ \
417	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
418	if (rcStrict2 == VINF_SUCCESS) \
419	{ /* likely */ } \
420	else \
421	return rcStrict2; \
422	} while (0)
423	#else
424	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
425	#endif /* IEM_WITH_SETJMP */
426
427
428	#ifndef IEM_WITH_SETJMP
429	/**
430	* Fetches the next signed byte from the opcode stream.
431	*
432	* @returns Strict VBox status code.
433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
434	* @param pi8 Where to return the signed byte.
435	*/
436	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
437	{
438	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
439	}
440	#endif /* !IEM_WITH_SETJMP */
441
442
443	/**
444	* Fetches the next signed byte from the opcode stream, returning automatically
445	* on failure.
446	*
447	* @param a_pi8 Where to return the signed byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	#ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
455	if (rcStrict2 != VINF_SUCCESS) \
456	return rcStrict2; \
457	} while (0)
458	#else /* IEM_WITH_SETJMP */
459	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
460
461	#endif /* IEM_WITH_SETJMP */
462
463
464	#ifndef IEM_WITH_SETJMP
465	/**
466	* Fetches the next signed byte from the opcode stream, extending it to
467	* unsigned 16-bit.
468	*
469	* @returns Strict VBox status code.
470	* @param pVCpu The cross context virtual CPU structure of the calling thread.
471	* @param pu16 Where to return the unsigned word.
472	*/
473	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
474	{
475	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
476	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
477	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
478
479	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
480	pVCpu->iem.s.offOpcode = offOpcode + 1;
481	return VINF_SUCCESS;
482	}
483	#endif /* !IEM_WITH_SETJMP */
484
485	/**
486	* Fetches the next signed byte from the opcode stream and sign-extending it to
487	* a word, returning automatically on failure.
488	*
489	* @param a_pu16 Where to return the word.
490	* @remark Implicitly references pVCpu.
491	*/
492	#ifndef IEM_WITH_SETJMP
493	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
494	do \
495	{ \
496	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
497	if (rcStrict2 != VINF_SUCCESS) \
498	return rcStrict2; \
499	} while (0)
500	#else
501	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
502	#endif
503
504	#ifndef IEM_WITH_SETJMP
505	/**
506	* Fetches the next signed byte from the opcode stream, extending it to
507	* unsigned 32-bit.
508	*
509	* @returns Strict VBox status code.
510	* @param pVCpu The cross context virtual CPU structure of the calling thread.
511	* @param pu32 Where to return the unsigned dword.
512	*/
513	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
514	{
515	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
516	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
517	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
518
519	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
520	pVCpu->iem.s.offOpcode = offOpcode + 1;
521	return VINF_SUCCESS;
522	}
523	#endif /* !IEM_WITH_SETJMP */
524
525	/**
526	* Fetches the next signed byte from the opcode stream and sign-extending it to
527	* a word, returning automatically on failure.
528	*
529	* @param a_pu32 Where to return the word.
530	* @remark Implicitly references pVCpu.
531	*/
532	#ifndef IEM_WITH_SETJMP
533	#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
534	do \
535	{ \
536	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
537	if (rcStrict2 != VINF_SUCCESS) \
538	return rcStrict2; \
539	} while (0)
540	#else
541	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
542	#endif
543
544
545	#ifndef IEM_WITH_SETJMP
546	/**
547	* Fetches the next signed byte from the opcode stream, extending it to
548	* unsigned 64-bit.
549	*
550	* @returns Strict VBox status code.
551	* @param pVCpu The cross context virtual CPU structure of the calling thread.
552	* @param pu64 Where to return the unsigned qword.
553	*/
554	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
555	{
556	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
557	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
558	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
559
560	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
561	pVCpu->iem.s.offOpcode = offOpcode + 1;
562	return VINF_SUCCESS;
563	}
564	#endif /* !IEM_WITH_SETJMP */
565
566	/**
567	* Fetches the next signed byte from the opcode stream and sign-extending it to
568	* a word, returning automatically on failure.
569	*
570	* @param a_pu64 Where to return the word.
571	* @remark Implicitly references pVCpu.
572	*/
573	#ifndef IEM_WITH_SETJMP
574	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
575	do \
576	{ \
577	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
578	if (rcStrict2 != VINF_SUCCESS) \
579	return rcStrict2; \
580	} while (0)
581	#else
582	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
583	#endif
584
585
586	#ifndef IEM_WITH_SETJMP
587	/**
588	* Fetches the next opcode byte.
589	*
590	* @returns Strict VBox status code.
591	* @param pVCpu The cross context virtual CPU structure of the
592	* calling thread.
593	* @param pu8 Where to return the opcode byte.
594	*/
595	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
596	{
597	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
598	pVCpu->iem.s.offModRm = offOpcode;
599	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
600	{
601	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
602	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
603	return VINF_SUCCESS;
604	}
605	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
606	}
607	#else /* IEM_WITH_SETJMP */
608	/**
609	* Fetches the next opcode byte, longjmp on error.
610	*
611	* @returns The opcode byte.
612	* @param pVCpu The cross context virtual CPU structure of the calling thread.
613	*/
614	DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
615	{
616	# ifdef IEM_WITH_CODE_TLB
617	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
618	pVCpu->iem.s.offModRm = offBuf;
619	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
620	if (RT_LIKELY( pbBuf != NULL
621	&& offBuf < pVCpu->iem.s.cbInstrBuf))
622	{
623	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
624	return pbBuf[offBuf];
625	}
626	# else
627	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
628	pVCpu->iem.s.offModRm = offOpcode;
629	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
630	{
631	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
632	return pVCpu->iem.s.abOpcode[offOpcode];
633	}
634	# endif
635	return iemOpcodeGetNextU8SlowJmp(pVCpu);
636	}
637	#endif /* IEM_WITH_SETJMP */
638
639	/**
640	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
641	* on failure.
642	*
643	* Will note down the position of the ModR/M byte for VT-x exits.
644	*
645	* @param a_pbRm Where to return the RM opcode byte.
646	* @remark Implicitly references pVCpu.
647	*/
648	#ifndef IEM_WITH_SETJMP
649	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
650	do \
651	{ \
652	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
653	if (rcStrict2 == VINF_SUCCESS) \
654	{ /* likely */ } \
655	else \
656	return rcStrict2; \
657	} while (0)
658	#else
659	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
660	#endif /* IEM_WITH_SETJMP */
661
662
663	#ifndef IEM_WITH_SETJMP
664
665	/**
666	* Fetches the next opcode word.
667	*
668	* @returns Strict VBox status code.
669	* @param pVCpu The cross context virtual CPU structure of the calling thread.
670	* @param pu16 Where to return the opcode word.
671	*/
672	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
673	{
674	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
675	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
676	{
677	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
678	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
679	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
680	# else
681	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
682	# endif
683	return VINF_SUCCESS;
684	}
685	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
686	}
687
688	#else /* IEM_WITH_SETJMP */
689
690	/**
691	* Fetches the next opcode word, longjmp on error.
692	*
693	* @returns The opcode word.
694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
695	*/
696	DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
697	{
698	# ifdef IEM_WITH_CODE_TLB
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
705	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
706	return (uint16_t const )&pbBuf[offBuf];
707	# else
708	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
709	# endif
710	}
711	# else
712	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
713	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
714	{
715	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
716	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
717	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
718	# else
719	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
720	# endif
721	}
722	# endif
723	return iemOpcodeGetNextU16SlowJmp(pVCpu);
724	}
725
726	#endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode word, returns automatically on failure.
730	*
731	* @param a_pu16 Where to return the opcode word.
732	* @remark Implicitly references pVCpu.
733	*/
734	#ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
739	if (rcStrict2 != VINF_SUCCESS) \
740	return rcStrict2; \
741	} while (0)
742	#else
743	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
744	#endif
745
746	#ifndef IEM_WITH_SETJMP
747	/**
748	* Fetches the next opcode word, zero extending it to a double word.
749	*
750	* @returns Strict VBox status code.
751	* @param pVCpu The cross context virtual CPU structure of the calling thread.
752	* @param pu32 Where to return the opcode double word.
753	*/
754	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
755	{
756	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
757	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
758	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
759
760	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
761	pVCpu->iem.s.offOpcode = offOpcode + 2;
762	return VINF_SUCCESS;
763	}
764	#endif /* !IEM_WITH_SETJMP */
765
766	/**
767	* Fetches the next opcode word and zero extends it to a double word, returns
768	* automatically on failure.
769	*
770	* @param a_pu32 Where to return the opcode double word.
771	* @remark Implicitly references pVCpu.
772	*/
773	#ifndef IEM_WITH_SETJMP
774	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
775	do \
776	{ \
777	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
778	if (rcStrict2 != VINF_SUCCESS) \
779	return rcStrict2; \
780	} while (0)
781	#else
782	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
783	#endif
784
785	#ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next opcode word, zero extending it to a quad word.
788	*
789	* @returns Strict VBox status code.
790	* @param pVCpu The cross context virtual CPU structure of the calling thread.
791	* @param pu64 Where to return the opcode quad word.
792	*/
793	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
794	{
795	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
796	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
797	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
798
799	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
800	pVCpu->iem.s.offOpcode = offOpcode + 2;
801	return VINF_SUCCESS;
802	}
803	#endif /* !IEM_WITH_SETJMP */
804
805	/**
806	* Fetches the next opcode word and zero extends it to a quad word, returns
807	* automatically on failure.
808	*
809	* @param a_pu64 Where to return the opcode quad word.
810	* @remark Implicitly references pVCpu.
811	*/
812	#ifndef IEM_WITH_SETJMP
813	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
814	do \
815	{ \
816	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
817	if (rcStrict2 != VINF_SUCCESS) \
818	return rcStrict2; \
819	} while (0)
820	#else
821	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
822	#endif
823
824
825	#ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed word from the opcode stream.
828	*
829	* @returns Strict VBox status code.
830	* @param pVCpu The cross context virtual CPU structure of the calling thread.
831	* @param pi16 Where to return the signed word.
832	*/
833	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
834	{
835	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
836	}
837	#endif /* !IEM_WITH_SETJMP */
838
839
840	/**
841	* Fetches the next signed word from the opcode stream, returning automatically
842	* on failure.
843	*
844	* @param a_pi16 Where to return the signed word.
845	* @remark Implicitly references pVCpu.
846	*/
847	#ifndef IEM_WITH_SETJMP
848	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
849	do \
850	{ \
851	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
852	if (rcStrict2 != VINF_SUCCESS) \
853	return rcStrict2; \
854	} while (0)
855	#else
856	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
857	#endif
858
859	#ifndef IEM_WITH_SETJMP
860
861	/**
862	* Fetches the next opcode dword.
863	*
864	* @returns Strict VBox status code.
865	* @param pVCpu The cross context virtual CPU structure of the calling thread.
866	* @param pu32 Where to return the opcode double word.
867	*/
868	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
869	{
870	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
871	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
872	{
873	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
874	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
875	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
876	# else
877	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
878	pVCpu->iem.s.abOpcode[offOpcode + 1],
879	pVCpu->iem.s.abOpcode[offOpcode + 2],
880	pVCpu->iem.s.abOpcode[offOpcode + 3]);
881	# endif
882	return VINF_SUCCESS;
883	}
884	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
885	}
886
887	#else /* IEM_WITH_SETJMP */
888
889	/**
890	* Fetches the next opcode dword, longjmp on error.
891	*
892	* @returns The opcode dword.
893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
894	*/
895	DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
896	{
897	# ifdef IEM_WITH_CODE_TLB
898	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
899	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
900	if (RT_LIKELY( pbBuf != NULL
901	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
902	{
903	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
904	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
905	return (uint32_t const )&pbBuf[offBuf];
906	# else
907	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
908	pbBuf[offBuf + 1],
909	pbBuf[offBuf + 2],
910	pbBuf[offBuf + 3]);
911	# endif
912	}
913	# else
914	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
915	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
916	{
917	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
918	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
919	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
920	# else
921	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
922	pVCpu->iem.s.abOpcode[offOpcode + 1],
923	pVCpu->iem.s.abOpcode[offOpcode + 2],
924	pVCpu->iem.s.abOpcode[offOpcode + 3]);
925	# endif
926	}
927	# endif
928	return iemOpcodeGetNextU32SlowJmp(pVCpu);
929	}
930
931	#endif /* IEM_WITH_SETJMP */
932
933	/**
934	* Fetches the next opcode dword, returns automatically on failure.
935	*
936	* @param a_pu32 Where to return the opcode dword.
937	* @remark Implicitly references pVCpu.
938	*/
939	#ifndef IEM_WITH_SETJMP
940	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
941	do \
942	{ \
943	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
944	if (rcStrict2 != VINF_SUCCESS) \
945	return rcStrict2; \
946	} while (0)
947	#else
948	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
949	#endif
950
951	#ifndef IEM_WITH_SETJMP
952	/**
953	* Fetches the next opcode dword, zero extending it to a quad word.
954	*
955	* @returns Strict VBox status code.
956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
957	* @param pu64 Where to return the opcode quad word.
958	*/
959	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
960	{
961	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
962	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
963	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
964
965	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
966	pVCpu->iem.s.abOpcode[offOpcode + 1],
967	pVCpu->iem.s.abOpcode[offOpcode + 2],
968	pVCpu->iem.s.abOpcode[offOpcode + 3]);
969	pVCpu->iem.s.offOpcode = offOpcode + 4;
970	return VINF_SUCCESS;
971	}
972	#endif /* !IEM_WITH_SETJMP */
973
974	/**
975	* Fetches the next opcode dword and zero extends it to a quad word, returns
976	* automatically on failure.
977	*
978	* @param a_pu64 Where to return the opcode quad word.
979	* @remark Implicitly references pVCpu.
980	*/
981	#ifndef IEM_WITH_SETJMP
982	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
983	do \
984	{ \
985	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
986	if (rcStrict2 != VINF_SUCCESS) \
987	return rcStrict2; \
988	} while (0)
989	#else
990	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
991	#endif
992
993
994	#ifndef IEM_WITH_SETJMP
995	/**
996	* Fetches the next signed double word from the opcode stream.
997	*
998	* @returns Strict VBox status code.
999	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1000	* @param pi32 Where to return the signed double word.
1001	*/
1002	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
1003	{
1004	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1005	}
1006	#endif
1007
1008	/**
1009	* Fetches the next signed double word from the opcode stream, returning
1010	* automatically on failure.
1011	*
1012	* @param a_pi32 Where to return the signed double word.
1013	* @remark Implicitly references pVCpu.
1014	*/
1015	#ifndef IEM_WITH_SETJMP
1016	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1017	do \
1018	{ \
1019	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1020	if (rcStrict2 != VINF_SUCCESS) \
1021	return rcStrict2; \
1022	} while (0)
1023	#else
1024	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1025	#endif
1026
1027	#ifndef IEM_WITH_SETJMP
1028	/**
1029	* Fetches the next opcode dword, sign extending it into a quad word.
1030	*
1031	* @returns Strict VBox status code.
1032	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1033	* @param pu64 Where to return the opcode quad word.
1034	*/
1035	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
1036	{
1037	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1038	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1039	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1040
1041	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1042	pVCpu->iem.s.abOpcode[offOpcode + 1],
1043	pVCpu->iem.s.abOpcode[offOpcode + 2],
1044	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1045	*pu64 = i32;
1046	pVCpu->iem.s.offOpcode = offOpcode + 4;
1047	return VINF_SUCCESS;
1048	}
1049	#endif /* !IEM_WITH_SETJMP */
1050
1051	/**
1052	* Fetches the next opcode double word and sign extends it to a quad word,
1053	* returns automatically on failure.
1054	*
1055	* @param a_pu64 Where to return the opcode quad word.
1056	* @remark Implicitly references pVCpu.
1057	*/
1058	#ifndef IEM_WITH_SETJMP
1059	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1060	do \
1061	{ \
1062	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1063	if (rcStrict2 != VINF_SUCCESS) \
1064	return rcStrict2; \
1065	} while (0)
1066	#else
1067	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1068	#endif
1069
1070	#ifndef IEM_WITH_SETJMP
1071
1072	/**
1073	* Fetches the next opcode qword.
1074	*
1075	* @returns Strict VBox status code.
1076	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1077	* @param pu64 Where to return the opcode qword.
1078	*/
1079	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
1080	{
1081	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1082	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1083	{
1084	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1085	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1086	# else
1087	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1088	pVCpu->iem.s.abOpcode[offOpcode + 1],
1089	pVCpu->iem.s.abOpcode[offOpcode + 2],
1090	pVCpu->iem.s.abOpcode[offOpcode + 3],
1091	pVCpu->iem.s.abOpcode[offOpcode + 4],
1092	pVCpu->iem.s.abOpcode[offOpcode + 5],
1093	pVCpu->iem.s.abOpcode[offOpcode + 6],
1094	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1095	# endif
1096	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1097	return VINF_SUCCESS;
1098	}
1099	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1100	}
1101
1102	#else /* IEM_WITH_SETJMP */
1103
1104	/**
1105	* Fetches the next opcode qword, longjmp on error.
1106	*
1107	* @returns The opcode qword.
1108	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1109	*/
1110	DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
1111	{
1112	# ifdef IEM_WITH_CODE_TLB
1113	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1114	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1115	if (RT_LIKELY( pbBuf != NULL
1116	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1117	{
1118	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1119	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1120	return (uint64_t const )&pbBuf[offBuf];
1121	# else
1122	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1123	pbBuf[offBuf + 1],
1124	pbBuf[offBuf + 2],
1125	pbBuf[offBuf + 3],
1126	pbBuf[offBuf + 4],
1127	pbBuf[offBuf + 5],
1128	pbBuf[offBuf + 6],
1129	pbBuf[offBuf + 7]);
1130	# endif
1131	}
1132	# else
1133	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1134	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1135	{
1136	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1137	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1138	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1139	# else
1140	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1141	pVCpu->iem.s.abOpcode[offOpcode + 1],
1142	pVCpu->iem.s.abOpcode[offOpcode + 2],
1143	pVCpu->iem.s.abOpcode[offOpcode + 3],
1144	pVCpu->iem.s.abOpcode[offOpcode + 4],
1145	pVCpu->iem.s.abOpcode[offOpcode + 5],
1146	pVCpu->iem.s.abOpcode[offOpcode + 6],
1147	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1148	# endif
1149	}
1150	# endif
1151	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1152	}
1153
1154	#endif /* IEM_WITH_SETJMP */
1155
1156	/**
1157	* Fetches the next opcode quad word, returns automatically on failure.
1158	*
1159	* @param a_pu64 Where to return the opcode quad word.
1160	* @remark Implicitly references pVCpu.
1161	*/
1162	#ifndef IEM_WITH_SETJMP
1163	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1164	do \
1165	{ \
1166	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1167	if (rcStrict2 != VINF_SUCCESS) \
1168	return rcStrict2; \
1169	} while (0)
1170	#else
1171	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1172	#endif
1173
1174
1175	/** @name Misc Worker Functions.
1176	* @{
1177	*/
1178
1179	/**
1180	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1181	* not (kind of obsolete now).
1182	*
1183	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1184	*/
1185	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1186
1187	/**
1188	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1189	*
1190	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1191	* @param a_fEfl The new EFLAGS.
1192	*/
1193	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1194
1195
1196	/**
1197	* Loads a NULL data selector into a selector register, both the hidden and
1198	* visible parts, in protected mode.
1199	*
1200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1201	* @param pSReg Pointer to the segment register.
1202	* @param uRpl The RPL.
1203	*/
1204	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
1205	{
1206	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1207	* data selector in protected mode. */
1208	pSReg->Sel = uRpl;
1209	pSReg->ValidSel = uRpl;
1210	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1211	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1212	{
1213	/* VT-x (Intel 3960x) observed doing something like this. */
1214	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1215	pSReg->u32Limit = UINT32_MAX;
1216	pSReg->u64Base = 0;
1217	}
1218	else
1219	{
1220	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1221	pSReg->u32Limit = 0;
1222	pSReg->u64Base = 0;
1223	}
1224	}
1225
1226	/** @} */
1227
1228
1229	/*
1230	*
1231	* Helpers routines.
1232	* Helpers routines.
1233	* Helpers routines.
1234	*
1235	*/
1236
1237	/**
1238	* Recalculates the effective operand size.
1239	*
1240	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1241	*/
1242	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu)
1243	{
1244	switch (pVCpu->iem.s.enmCpuMode)
1245	{
1246	case IEMMODE_16BIT:
1247	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1248	break;
1249	case IEMMODE_32BIT:
1250	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1251	break;
1252	case IEMMODE_64BIT:
1253	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1254	{
1255	case 0:
1256	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1257	break;
1258	case IEM_OP_PRF_SIZE_OP:
1259	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1260	break;
1261	case IEM_OP_PRF_SIZE_REX_W:
1262	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1263	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1264	break;
1265	}
1266	break;
1267	default:
1268	AssertFailed();
1269	}
1270	}
1271
1272
1273	/**
1274	* Sets the default operand size to 64-bit and recalculates the effective
1275	* operand size.
1276	*
1277	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1278	*/
1279	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
1280	{
1281	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1282	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1283	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1284	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1285	else
1286	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1287	}
1288
1289
1290
1291
1292	/** @name Register Access.
1293	* @{
1294	*/
1295
1296	/**
1297	* Gets a reference (pointer) to the specified hidden segment register.
1298	*
1299	* @returns Hidden register reference.
1300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1301	* @param iSegReg The segment register.
1302	*/
1303	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
1304	{
1305	Assert(iSegReg < X86_SREG_COUNT);
1306	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1307	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1308
1309	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1310	return pSReg;
1311	}
1312
1313
1314	/**
1315	* Ensures that the given hidden segment register is up to date.
1316	*
1317	* @returns Hidden register reference.
1318	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1319	* @param pSReg The segment register.
1320	*/
1321	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1322	{
1323	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1324	NOREF(pVCpu);
1325	return pSReg;
1326	}
1327
1328
1329	/**
1330	* Gets a reference (pointer) to the specified segment register (the selector
1331	* value).
1332	*
1333	* @returns Pointer to the selector variable.
1334	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1335	* @param iSegReg The segment register.
1336	*/
1337	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
1338	{
1339	Assert(iSegReg < X86_SREG_COUNT);
1340	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1341	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1342	}
1343
1344
1345	/**
1346	* Fetches the selector value of a segment register.
1347	*
1348	* @returns The selector value.
1349	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1350	* @param iSegReg The segment register.
1351	*/
1352	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
1353	{
1354	Assert(iSegReg < X86_SREG_COUNT);
1355	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1356	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1357	}
1358
1359
1360	/**
1361	* Fetches the base address value of a segment register.
1362	*
1363	* @returns The selector value.
1364	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1365	* @param iSegReg The segment register.
1366	*/
1367	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1368	{
1369	Assert(iSegReg < X86_SREG_COUNT);
1370	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1371	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1372	}
1373
1374
1375	/**
1376	* Gets a reference (pointer) to the specified general purpose register.
1377	*
1378	* @returns Register reference.
1379	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1380	* @param iReg The general purpose register.
1381	*/
1382	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
1383	{
1384	Assert(iReg < 16);
1385	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1386	}
1387
1388
1389	/**
1390	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1391	*
1392	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1393	*
1394	* @returns Register reference.
1395	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1396	* @param iReg The register.
1397	*/
1398	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
1399	{
1400	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1401	{
1402	Assert(iReg < 16);
1403	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1404	}
1405	/* high 8-bit register. */
1406	Assert(iReg < 8);
1407	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1408	}
1409
1410
1411	/**
1412	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1413	*
1414	* @returns Register reference.
1415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1416	* @param iReg The register.
1417	*/
1418	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
1419	{
1420	Assert(iReg < 16);
1421	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1422	}
1423
1424
1425	/**
1426	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1427	*
1428	* @returns Register reference.
1429	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1430	* @param iReg The register.
1431	*/
1432	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
1433	{
1434	Assert(iReg < 16);
1435	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1436	}
1437
1438
1439	/**
1440	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1441	*
1442	* @returns Register reference.
1443	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1444	* @param iReg The register.
1445	*/
1446	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg)
1447	{
1448	Assert(iReg < 16);
1449	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1450	}
1451
1452
1453	/**
1454	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1455	*
1456	* @returns Register reference.
1457	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1458	* @param iReg The register.
1459	*/
1460	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
1461	{
1462	Assert(iReg < 64);
1463	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1464	}
1465
1466
1467	/**
1468	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1469	*
1470	* @returns Register reference.
1471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1472	* @param iReg The register.
1473	*/
1474	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg)
1475	{
1476	Assert(iReg < 16);
1477	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1478	}
1479
1480
1481	/**
1482	* Gets a reference (pointer) to the specified segment register's base address.
1483	*
1484	* @returns Segment register base address reference.
1485	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1486	* @param iSegReg The segment selector.
1487	*/
1488	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1489	{
1490	Assert(iSegReg < X86_SREG_COUNT);
1491	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1492	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1493	}
1494
1495
1496	/**
1497	* Fetches the value of a 8-bit general purpose register.
1498	*
1499	* @returns The register value.
1500	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1501	* @param iReg The register.
1502	*/
1503	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
1504	{
1505	return *iemGRegRefU8(pVCpu, iReg);
1506	}
1507
1508
1509	/**
1510	* Fetches the value of a 16-bit general purpose register.
1511	*
1512	* @returns The register value.
1513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1514	* @param iReg The register.
1515	*/
1516	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
1517	{
1518	Assert(iReg < 16);
1519	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1520	}
1521
1522
1523	/**
1524	* Fetches the value of a 32-bit general purpose register.
1525	*
1526	* @returns The register value.
1527	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1528	* @param iReg The register.
1529	*/
1530	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
1531	{
1532	Assert(iReg < 16);
1533	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1534	}
1535
1536
1537	/**
1538	* Fetches the value of a 64-bit general purpose register.
1539	*
1540	* @returns The register value.
1541	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1542	* @param iReg The register.
1543	*/
1544	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
1545	{
1546	Assert(iReg < 16);
1547	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1548	}
1549
1550
1551	/**
1552	* Get the address of the top of the stack.
1553	*
1554	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1555	*/
1556	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
1557	{
1558	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1559	return pVCpu->cpum.GstCtx.rsp;
1560	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1561	return pVCpu->cpum.GstCtx.esp;
1562	return pVCpu->cpum.GstCtx.sp;
1563	}
1564
1565	#if 0 /* unused and buggy */
1566	/**
1567	* Updates the RIP/EIP/IP to point to the next instruction.
1568	*
1569	* This function leaves the EFLAGS.RF flag alone.
1570	*
1571	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1572	* @param cbInstr The number of bytes to add.
1573	*/
1574	DECLINLINE(void) iemRegAddToRipKeepRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1575	{
1576	switch (pVCpu->iem.s.enmCpuMode)
1577	{
1578	case IEMMODE_16BIT:
1579	Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
1580	pVCpu->cpum.GstCtx.eip += cbInstr;
1581	pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
1582	break;
1583
1584	case IEMMODE_32BIT:
1585	pVCpu->cpum.GstCtx.eip += cbInstr;
1586	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1587	break;
1588
1589	case IEMMODE_64BIT:
1590	pVCpu->cpum.GstCtx.rip += cbInstr;
1591	break;
1592	default: AssertFailed();
1593	}
1594	}
1595	#endif
1596
1597	#if 0
1598	/**
1599	* Updates the RIP/EIP/IP to point to the next instruction.
1600	*
1601	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1602	*/
1603	DECLINLINE(void) iemRegUpdateRipKeepRF(PVMCPUCC pVCpu)
1604	{
1605	return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1606	}
1607	#endif
1608
1609	/**
1610	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
1611	* and CPUMCTX_INHIBIT_SHADOW.
1612	*
1613	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1614	* @param cbInstr The number of bytes to add.
1615	*/
1616	DECLINLINE(void) iemRegAddToRipAndClearRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1617	{
1618	/*
1619	* Advance RIP.
1620	*
1621	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1622	* while in all other modes except LM64 the updates are 32-bit. This means
1623	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1624	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1625	*
1626	* See PC wrap around tests in bs3-cpu-weird-1.
1627	*/
1628	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1629	uint64_t const uRipNext = uRipPrev + cbInstr;
1630	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1631	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
1632	pVCpu->cpum.GstCtx.rip = uRipNext;
1633	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1634	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1635	else
1636	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1637
1638	/*
1639	* Clear RF and interrupt shadowing.
1640	*/
1641	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1642	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1643	}
1644
1645
1646	/**
1647	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1648	*
1649	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1650	*/
1651	DECLINLINE(void) iemRegUpdateRipAndClearRF(PVMCPUCC pVCpu)
1652	{
1653	return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1654	}
1655
1656
1657	/**
1658	* Adds to the stack pointer.
1659	*
1660	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1661	* @param cbToAdd The number of bytes to add (8-bit!).
1662	*/
1663	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
1664	{
1665	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1666	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1667	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1668	pVCpu->cpum.GstCtx.esp += cbToAdd;
1669	else
1670	pVCpu->cpum.GstCtx.sp += cbToAdd;
1671	}
1672
1673
1674	/**
1675	* Subtracts from the stack pointer.
1676	*
1677	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1678	* @param cbToSub The number of bytes to subtract (8-bit!).
1679	*/
1680	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
1681	{
1682	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1683	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1684	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1685	pVCpu->cpum.GstCtx.esp -= cbToSub;
1686	else
1687	pVCpu->cpum.GstCtx.sp -= cbToSub;
1688	}
1689
1690
1691	/**
1692	* Adds to the temporary stack pointer.
1693	*
1694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1695	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1696	* @param cbToAdd The number of bytes to add (16-bit).
1697	*/
1698	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
1699	{
1700	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1701	pTmpRsp->u += cbToAdd;
1702	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1703	pTmpRsp->DWords.dw0 += cbToAdd;
1704	else
1705	pTmpRsp->Words.w0 += cbToAdd;
1706	}
1707
1708
1709	/**
1710	* Subtracts from the temporary stack pointer.
1711	*
1712	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1713	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1714	* @param cbToSub The number of bytes to subtract.
1715	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1716	* expecting that.
1717	*/
1718	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
1719	{
1720	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1721	pTmpRsp->u -= cbToSub;
1722	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1723	pTmpRsp->DWords.dw0 -= cbToSub;
1724	else
1725	pTmpRsp->Words.w0 -= cbToSub;
1726	}
1727
1728
1729	/**
1730	* Calculates the effective stack address for a push of the specified size as
1731	* well as the new RSP value (upper bits may be masked).
1732	*
1733	* @returns Effective stack addressf for the push.
1734	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1735	* @param cbItem The size of the stack item to pop.
1736	* @param puNewRsp Where to return the new RSP value.
1737	*/
1738	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1739	{
1740	RTUINT64U uTmpRsp;
1741	RTGCPTR GCPtrTop;
1742	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1743
1744	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1745	GCPtrTop = uTmpRsp.u -= cbItem;
1746	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1747	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
1748	else
1749	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
1750	*puNewRsp = uTmpRsp.u;
1751	return GCPtrTop;
1752	}
1753
1754
1755	/**
1756	* Gets the current stack pointer and calculates the value after a pop of the
1757	* specified size.
1758	*
1759	* @returns Current stack pointer.
1760	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1761	* @param cbItem The size of the stack item to pop.
1762	* @param puNewRsp Where to return the new RSP value.
1763	*/
1764	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1765	{
1766	RTUINT64U uTmpRsp;
1767	RTGCPTR GCPtrTop;
1768	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1769
1770	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1771	{
1772	GCPtrTop = uTmpRsp.u;
1773	uTmpRsp.u += cbItem;
1774	}
1775	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1776	{
1777	GCPtrTop = uTmpRsp.DWords.dw0;
1778	uTmpRsp.DWords.dw0 += cbItem;
1779	}
1780	else
1781	{
1782	GCPtrTop = uTmpRsp.Words.w0;
1783	uTmpRsp.Words.w0 += cbItem;
1784	}
1785	*puNewRsp = uTmpRsp.u;
1786	return GCPtrTop;
1787	}
1788
1789
1790	/**
1791	* Calculates the effective stack address for a push of the specified size as
1792	* well as the new temporary RSP value (upper bits may be masked).
1793	*
1794	* @returns Effective stack addressf for the push.
1795	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1796	* @param pTmpRsp The temporary stack pointer. This is updated.
1797	* @param cbItem The size of the stack item to pop.
1798	*/
1799	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1800	{
1801	RTGCPTR GCPtrTop;
1802
1803	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1804	GCPtrTop = pTmpRsp->u -= cbItem;
1805	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1806	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
1807	else
1808	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
1809	return GCPtrTop;
1810	}
1811
1812
1813	/**
1814	* Gets the effective stack address for a pop of the specified size and
1815	* calculates and updates the temporary RSP.
1816	*
1817	* @returns Current stack pointer.
1818	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1819	* @param pTmpRsp The temporary stack pointer. This is updated.
1820	* @param cbItem The size of the stack item to pop.
1821	*/
1822	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1823	{
1824	RTGCPTR GCPtrTop;
1825	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1826	{
1827	GCPtrTop = pTmpRsp->u;
1828	pTmpRsp->u += cbItem;
1829	}
1830	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1831	{
1832	GCPtrTop = pTmpRsp->DWords.dw0;
1833	pTmpRsp->DWords.dw0 += cbItem;
1834	}
1835	else
1836	{
1837	GCPtrTop = pTmpRsp->Words.w0;
1838	pTmpRsp->Words.w0 += cbItem;
1839	}
1840	return GCPtrTop;
1841	}
1842
1843	/** @} */
1844
1845
1846	/** @name FPU access and helpers.
1847	*
1848	* @{
1849	*/
1850
1851
1852	/**
1853	* Hook for preparing to use the host FPU.
1854	*
1855	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1856	*
1857	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1858	*/
1859	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
1860	{
1861	#ifdef IN_RING3
1862	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1863	#else
1864	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
1865	#endif
1866	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1867	}
1868
1869
1870	/**
1871	* Hook for preparing to use the host FPU for SSE.
1872	*
1873	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1874	*
1875	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1876	*/
1877	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
1878	{
1879	iemFpuPrepareUsage(pVCpu);
1880	}
1881
1882
1883	/**
1884	* Hook for preparing to use the host FPU for AVX.
1885	*
1886	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1887	*
1888	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1889	*/
1890	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
1891	{
1892	iemFpuPrepareUsage(pVCpu);
1893	}
1894
1895
1896	/**
1897	* Hook for actualizing the guest FPU state before the interpreter reads it.
1898	*
1899	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1900	*
1901	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1902	*/
1903	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
1904	{
1905	#ifdef IN_RING3
1906	NOREF(pVCpu);
1907	#else
1908	CPUMRZFpuStateActualizeForRead(pVCpu);
1909	#endif
1910	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1911	}
1912
1913
1914	/**
1915	* Hook for actualizing the guest FPU state before the interpreter changes it.
1916	*
1917	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1918	*
1919	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1920	*/
1921	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
1922	{
1923	#ifdef IN_RING3
1924	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1925	#else
1926	CPUMRZFpuStateActualizeForChange(pVCpu);
1927	#endif
1928	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1929	}
1930
1931
1932	/**
1933	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
1934	* only.
1935	*
1936	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1937	*
1938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1939	*/
1940	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
1941	{
1942	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
1943	NOREF(pVCpu);
1944	#else
1945	CPUMRZFpuStateActualizeSseForRead(pVCpu);
1946	#endif
1947	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1948	}
1949
1950
1951	/**
1952	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
1953	* read+write.
1954	*
1955	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1956	*
1957	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1958	*/
1959	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
1960	{
1961	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
1962	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1963	#else
1964	CPUMRZFpuStateActualizeForChange(pVCpu);
1965	#endif
1966	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1967
1968	/* Make sure any changes are loaded the next time around. */
1969	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
1970	}
1971
1972
1973	/**
1974	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
1975	* only.
1976	*
1977	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1978	*
1979	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1980	*/
1981	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
1982	{
1983	#ifdef IN_RING3
1984	NOREF(pVCpu);
1985	#else
1986	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
1987	#endif
1988	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1989	}
1990
1991
1992	/**
1993	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
1994	* read+write.
1995	*
1996	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1997	*
1998	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1999	*/
2000	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
2001	{
2002	#ifdef IN_RING3
2003	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2004	#else
2005	CPUMRZFpuStateActualizeForChange(pVCpu);
2006	#endif
2007	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2008
2009	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2010	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2011	}
2012
2013
2014	/**
2015	* Stores a QNaN value into a FPU register.
2016	*
2017	* @param pReg Pointer to the register.
2018	*/
2019	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
2020	{
2021	pReg->au32[0] = UINT32_C(0x00000000);
2022	pReg->au32[1] = UINT32_C(0xc0000000);
2023	pReg->au16[4] = UINT16_C(0xffff);
2024	}
2025
2026
2027	/**
2028	* Updates the FOP, FPU.CS and FPUIP registers.
2029	*
2030	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2031	* @param pFpuCtx The FPU context.
2032	*/
2033	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
2034	{
2035	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2036	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
2037	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2038	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2039	{
2040	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2041	* happens in real mode here based on the fnsave and fnstenv images. */
2042	pFpuCtx->CS = 0;
2043	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2044	}
2045	else if (!IEM_IS_LONG_MODE(pVCpu))
2046	{
2047	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2048	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2049	}
2050	else
2051	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2052	}
2053
2054
2055
2056
2057
2058	/**
2059	* Marks the specified stack register as free (for FFREE).
2060	*
2061	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2062	* @param iStReg The register to free.
2063	*/
2064	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
2065	{
2066	Assert(iStReg < 8);
2067	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2068	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2069	pFpuCtx->FTW &= ~RT_BIT(iReg);
2070	}
2071
2072
2073	/**
2074	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2075	*
2076	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2077	*/
2078	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu)
2079	{
2080	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2081	uint16_t uFsw = pFpuCtx->FSW;
2082	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2083	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2084	uFsw &= ~X86_FSW_TOP_MASK;
2085	uFsw \|= uTop;
2086	pFpuCtx->FSW = uFsw;
2087	}
2088
2089
2090	/**
2091	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2092	*
2093	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2094	*/
2095	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu)
2096	{
2097	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2098	uint16_t uFsw = pFpuCtx->FSW;
2099	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2100	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2101	uFsw &= ~X86_FSW_TOP_MASK;
2102	uFsw \|= uTop;
2103	pFpuCtx->FSW = uFsw;
2104	}
2105
2106
2107
2108
2109	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
2110	{
2111	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2112	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2113	if (pFpuCtx->FTW & RT_BIT(iReg))
2114	return VINF_SUCCESS;
2115	return VERR_NOT_FOUND;
2116	}
2117
2118
2119	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
2120	{
2121	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2122	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2123	if (pFpuCtx->FTW & RT_BIT(iReg))
2124	{
2125	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2126	return VINF_SUCCESS;
2127	}
2128	return VERR_NOT_FOUND;
2129	}
2130
2131
2132	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2133	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
2134	{
2135	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2136	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2137	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2138	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2139	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2140	{
2141	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2142	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2143	return VINF_SUCCESS;
2144	}
2145	return VERR_NOT_FOUND;
2146	}
2147
2148
2149	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
2150	{
2151	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2152	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2153	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2154	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2155	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2156	{
2157	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2158	return VINF_SUCCESS;
2159	}
2160	return VERR_NOT_FOUND;
2161	}
2162
2163
2164	/**
2165	* Rotates the stack registers when setting new TOS.
2166	*
2167	* @param pFpuCtx The FPU context.
2168	* @param iNewTop New TOS value.
2169	* @remarks We only do this to speed up fxsave/fxrstor which
2170	* arrange the FP registers in stack order.
2171	* MUST be done before writing the new TOS (FSW).
2172	*/
2173	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop)
2174	{
2175	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2176	RTFLOAT80U ar80Temp[8];
2177
2178	if (iOldTop == iNewTop)
2179	return;
2180
2181	/* Unscrew the stack and get it into 'native' order. */
2182	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2183	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2184	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2185	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2186	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2187	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2188	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2189	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2190
2191	/* Now rotate the stack to the new position. */
2192	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2193	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2194	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2195	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2196	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2197	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2198	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2199	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2200	}
2201
2202
2203	/**
2204	* Updates the FPU exception status after FCW is changed.
2205	*
2206	* @param pFpuCtx The FPU context.
2207	*/
2208	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
2209	{
2210	uint16_t u16Fsw = pFpuCtx->FSW;
2211	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2212	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2213	else
2214	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2215	pFpuCtx->FSW = u16Fsw;
2216	}
2217
2218
2219	/**
2220	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2221	*
2222	* @returns The full FTW.
2223	* @param pFpuCtx The FPU context.
2224	*/
2225	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
2226	{
2227	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2228	uint16_t u16Ftw = 0;
2229	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2230	for (unsigned iSt = 0; iSt < 8; iSt++)
2231	{
2232	unsigned const iReg = (iSt + iTop) & 7;
2233	if (!(u8Ftw & RT_BIT(iReg)))
2234	u16Ftw \|= 3 << (iReg * 2); /* empty */
2235	else
2236	{
2237	uint16_t uTag;
2238	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2239	if (pr80Reg->s.uExponent == 0x7fff)
2240	uTag = 2; /* Exponent is all 1's => Special. */
2241	else if (pr80Reg->s.uExponent == 0x0000)
2242	{
2243	if (pr80Reg->s.uMantissa == 0x0000)
2244	uTag = 1; /* All bits are zero => Zero. */
2245	else
2246	uTag = 2; /* Must be special. */
2247	}
2248	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2249	uTag = 0; /* Valid. */
2250	else
2251	uTag = 2; /* Must be special. */
2252
2253	u16Ftw \|= uTag << (iReg * 2);
2254	}
2255	}
2256
2257	return u16Ftw;
2258	}
2259
2260
2261	/**
2262	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2263	*
2264	* @returns The compressed FTW.
2265	* @param u16FullFtw The full FTW to convert.
2266	*/
2267	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw)
2268	{
2269	uint8_t u8Ftw = 0;
2270	for (unsigned i = 0; i < 8; i++)
2271	{
2272	if ((u16FullFtw & 3) != 3 /empty/)
2273	u8Ftw \|= RT_BIT(i);
2274	u16FullFtw >>= 2;
2275	}
2276
2277	return u8Ftw;
2278	}
2279
2280	/** @} */
2281
2282
2283	/** @name Memory access.
2284	*
2285	* @{
2286	*/
2287
2288
2289	/**
2290	* Checks whether alignment checks are enabled or not.
2291	*
2292	* @returns true if enabled, false if not.
2293	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2294	*/
2295	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu)
2296	{
2297	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2298	return pVCpu->iem.s.uCpl == 3
2299	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2300	}
2301
2302	/**
2303	* Checks if the given segment can be written to, raise the appropriate
2304	* exception if not.
2305	*
2306	* @returns VBox strict status code.
2307	*
2308	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2309	* @param pHid Pointer to the hidden register.
2310	* @param iSegReg The register number.
2311	* @param pu64BaseAddr Where to return the base address to use for the
2312	* segment. (In 64-bit code it may differ from the
2313	* base in the hidden segment.)
2314	*/
2315	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2316	{
2317	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2318
2319	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2320	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2321	else
2322	{
2323	if (!pHid->Attr.n.u1Present)
2324	{
2325	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2326	AssertRelease(uSel == 0);
2327	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2328	return iemRaiseGeneralProtectionFault0(pVCpu);
2329	}
2330
2331	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2332	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2333	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2334	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2335	*pu64BaseAddr = pHid->u64Base;
2336	}
2337	return VINF_SUCCESS;
2338	}
2339
2340
2341	/**
2342	* Checks if the given segment can be read from, raise the appropriate
2343	* exception if not.
2344	*
2345	* @returns VBox strict status code.
2346	*
2347	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2348	* @param pHid Pointer to the hidden register.
2349	* @param iSegReg The register number.
2350	* @param pu64BaseAddr Where to return the base address to use for the
2351	* segment. (In 64-bit code it may differ from the
2352	* base in the hidden segment.)
2353	*/
2354	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2355	{
2356	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2357
2358	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2359	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2360	else
2361	{
2362	if (!pHid->Attr.n.u1Present)
2363	{
2364	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2365	AssertRelease(uSel == 0);
2366	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2367	return iemRaiseGeneralProtectionFault0(pVCpu);
2368	}
2369
2370	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2371	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2372	*pu64BaseAddr = pHid->u64Base;
2373	}
2374	return VINF_SUCCESS;
2375	}
2376
2377
2378	/**
2379	* Maps a physical page.
2380	*
2381	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2382	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2383	* @param GCPhysMem The physical address.
2384	* @param fAccess The intended access.
2385	* @param ppvMem Where to return the mapping address.
2386	* @param pLock The PGM lock.
2387	*/
2388	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
2389	{
2390	#ifdef IEM_LOG_MEMORY_WRITES
2391	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2392	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2393	#endif
2394
2395	/** @todo This API may require some improving later. A private deal with PGM
2396	* regarding locking and unlocking needs to be struct. A couple of TLBs
2397	* living in PGM, but with publicly accessible inlined access methods
2398	* could perhaps be an even better solution. */
2399	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2400	GCPhysMem,
2401	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2402	pVCpu->iem.s.fBypassHandlers,
2403	ppvMem,
2404	pLock);
2405	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2406	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2407
2408	return rc;
2409	}
2410
2411
2412	/**
2413	* Unmap a page previously mapped by iemMemPageMap.
2414	*
2415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2416	* @param GCPhysMem The physical address.
2417	* @param fAccess The intended access.
2418	* @param pvMem What iemMemPageMap returned.
2419	* @param pLock The PGM lock.
2420	*/
2421	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
2422	{
2423	NOREF(pVCpu);
2424	NOREF(GCPhysMem);
2425	NOREF(fAccess);
2426	NOREF(pvMem);
2427	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2428	}
2429
2430	#ifdef IEM_WITH_SETJMP
2431
2432	/** @todo slim this down */
2433	DECLINLINE(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2434	{
2435	Assert(cbMem >= 1);
2436	Assert(iSegReg < X86_SREG_COUNT);
2437
2438	/*
2439	* 64-bit mode is simpler.
2440	*/
2441	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2442	{
2443	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2444	{
2445	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2446	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2447	GCPtrMem += pSel->u64Base;
2448	}
2449
2450	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2451	return GCPtrMem;
2452	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2453	}
2454	/*
2455	* 16-bit and 32-bit segmentation.
2456	*/
2457	else if (iSegReg != UINT8_MAX)
2458	{
2459	/** @todo Does this apply to segments with 4G-1 limit? */
2460	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2461	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2462	{
2463	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2464	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2465	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2466	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2467	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2468	\| X86_SEL_TYPE_CODE))
2469	{
2470	case X86DESCATTR_P: /* readonly data, expand up */
2471	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2472	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2473	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2474	/* expand up */
2475	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2476	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2477	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2478	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2479	break;
2480
2481	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2482	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2483	/* expand down */
2484	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2485	&& ( pSel->Attr.n.u1DefBig
2486	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2487	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2488	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2489	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2490	break;
2491
2492	default:
2493	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2494	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2495	break;
2496	}
2497	}
2498	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2499	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2500	}
2501	/*
2502	* 32-bit flat address.
2503	*/
2504	else
2505	return GCPtrMem;
2506	}
2507
2508
2509	/** @todo slim this down */
2510	DECLINLINE(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2511	{
2512	Assert(cbMem >= 1);
2513	Assert(iSegReg < X86_SREG_COUNT);
2514
2515	/*
2516	* 64-bit mode is simpler.
2517	*/
2518	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2519	{
2520	if (iSegReg >= X86_SREG_FS)
2521	{
2522	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2523	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2524	GCPtrMem += pSel->u64Base;
2525	}
2526
2527	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2528	return GCPtrMem;
2529	}
2530	/*
2531	* 16-bit and 32-bit segmentation.
2532	*/
2533	else
2534	{
2535	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2536	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2537	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2538	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2539	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2540	{
2541	/* expand up */
2542	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2543	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2544	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2545	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2546	}
2547	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2548	{
2549	/* expand down */
2550	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2551	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2552	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2553	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2554	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2555	}
2556	else
2557	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2558	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2559	}
2560	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2561	}
2562
2563	#endif /* IEM_WITH_SETJMP */
2564
2565	/**
2566	* Fakes a long mode stack selector for SS = 0.
2567	*
2568	* @param pDescSs Where to return the fake stack descriptor.
2569	* @param uDpl The DPL we want.
2570	*/
2571	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
2572	{
2573	pDescSs->Long.au64[0] = 0;
2574	pDescSs->Long.au64[1] = 0;
2575	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2576	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2577	pDescSs->Long.Gen.u2Dpl = uDpl;
2578	pDescSs->Long.Gen.u1Present = 1;
2579	pDescSs->Long.Gen.u1Long = 1;
2580	}
2581
2582	/** @} */
2583
2584
2585	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2586
2587	/**
2588	* Gets CR0 fixed-0 bits in VMX non-root mode.
2589	*
2590	* We do this rather than fetching what we report to the guest (in
2591	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2592	* values regardless of whether unrestricted-guest feature is available on the CPU.
2593	*
2594	* @returns CR0 fixed-0 bits.
2595	* @param pVCpu The cross context virtual CPU structure.
2596	*/
2597	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu)
2598	{
2599	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2600	Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
2601
2602	static uint64_t const s_auCr0Fixed0[2] = { VMX_V_CR0_FIXED0, VMX_V_CR0_FIXED0_UX };
2603	PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
2604	uint8_t const fUnrestrictedGuest = !!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
2605	uint64_t const uCr0Fixed0 = s_auCr0Fixed0[fUnrestrictedGuest];
2606	Assert(!(uCr0Fixed0 & (X86_CR0_NW \| X86_CR0_CD)));
2607	return uCr0Fixed0;
2608	}
2609
2610
2611	/**
2612	* Sets virtual-APIC write emulation as pending.
2613	*
2614	* @param pVCpu The cross context virtual CPU structure.
2615	* @param offApic The offset in the virtual-APIC page that was written.
2616	*/
2617	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic)
2618	{
2619	Assert(offApic < XAPIC_OFF_END + 4);
2620
2621	/*
2622	* Record the currently updated APIC offset, as we need this later for figuring
2623	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2624	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2625	*/
2626	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2627
2628	/*
2629	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2630	* virtualization or APIC-write emulation).
2631	*/
2632	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2633	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2634	}
2635
2636	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2637
2638	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 97334

Download in other formats: