IEMInline.h@ 97406

Last change on this file since 97406 was 97406, checked in by vboxsync, 2 years ago
VMM/IEM,CPUM: Partial single stepping support in the interpreter. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 92.1 KB

Line
1	/* $Id: IEMInline.h 97406 2022-11-05 12:42:14Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184
185	/**
186	* Initializes the execution state.
187	*
188	* @param pVCpu The cross context virtual CPU structure of the
189	* calling thread.
190	* @param fBypassHandlers Whether to bypass access handlers.
191	*
192	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
193	* side-effects in strict builds.
194	*/
195	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
196	{
197	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
198	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
199	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
200	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
207
208	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
209	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
210	#ifdef VBOX_STRICT
211	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
212	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
213	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
215	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
216	pVCpu->iem.s.uRexReg = 127;
217	pVCpu->iem.s.uRexB = 127;
218	pVCpu->iem.s.offModRm = 127;
219	pVCpu->iem.s.uRexIndex = 127;
220	pVCpu->iem.s.iEffSeg = 127;
221	pVCpu->iem.s.idxPrefix = 127;
222	pVCpu->iem.s.uVex3rdReg = 127;
223	pVCpu->iem.s.uVexLength = 127;
224	pVCpu->iem.s.fEvexStuff = 127;
225	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
226	# ifdef IEM_WITH_CODE_TLB
227	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
228	pVCpu->iem.s.pbInstrBuf = NULL;
229	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
230	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
231	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
232	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
233	# else
234	pVCpu->iem.s.offOpcode = 127;
235	pVCpu->iem.s.cbOpcode = 127;
236	# endif
237	#endif
238
239	pVCpu->iem.s.cActiveMappings = 0;
240	pVCpu->iem.s.iNextMapping = 0;
241	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
242	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
243	#if 0
244	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
245	if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
246	&& CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
247	{
248	PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
249	Assert(pVmcs);
250	RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
251	if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
252	{
253	int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
254	pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
255	NIL_RTR0PTR /* pvUserR0 /, NIL_RTRCPTR / pvUserRC /, NULL / pszDesc */);
256	AssertRC(rc);
257	}
258	}
259	#endif
260	#endif
261	}
262
263	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
264	/**
265	* Performs a minimal reinitialization of the execution state.
266	*
267	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
268	* 'world-switch' types operations on the CPU. Currently only nested
269	* hardware-virtualization uses it.
270	*
271	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
272	*/
273	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu)
274	{
275	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
276	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
277
278	pVCpu->iem.s.uCpl = uCpl;
279	pVCpu->iem.s.enmCpuMode = enmMode;
280	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
281	pVCpu->iem.s.enmEffAddrMode = enmMode;
282	if (enmMode != IEMMODE_64BIT)
283	{
284	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	else
288	{
289	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
290	pVCpu->iem.s.enmEffOpSize = enmMode;
291	}
292	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
293	#ifndef IEM_WITH_CODE_TLB
294	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
295	pVCpu->iem.s.offOpcode = 0;
296	pVCpu->iem.s.cbOpcode = 0;
297	#endif
298	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
299	}
300	#endif
301
302	/**
303	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
304	*
305	* @param pVCpu The cross context virtual CPU structure of the
306	* calling thread.
307	*/
308	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
309	{
310	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
311	#ifdef VBOX_STRICT
312	# ifdef IEM_WITH_CODE_TLB
313	NOREF(pVCpu);
314	# else
315	pVCpu->iem.s.cbOpcode = 0;
316	# endif
317	#else
318	NOREF(pVCpu);
319	#endif
320	}
321
322
323	/**
324	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
325	*
326	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
327	*
328	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
329	* @param pVCpu The cross context virtual CPU structure of the calling thread.
330	* @param rcStrict The status code to fiddle.
331	*/
332	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
333	{
334	iemUninitExec(pVCpu);
335	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
336	}
337
338
339	/**
340	* Macro used by the IEMExec* method to check the given instruction length.
341	*
342	* Will return on failure!
343	*
344	* @param a_cbInstr The given instruction length.
345	* @param a_cbMin The minimum length.
346	*/
347	#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
348	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
349	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
350
351
352
353	#ifndef IEM_WITH_SETJMP
354
355	/**
356	* Fetches the next opcode byte.
357	*
358	* @returns Strict VBox status code.
359	* @param pVCpu The cross context virtual CPU structure of the
360	* calling thread.
361	* @param pu8 Where to return the opcode byte.
362	*/
363	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
364	{
365	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
366	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
367	{
368	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
369	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
370	return VINF_SUCCESS;
371	}
372	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
373	}
374
375	#else /* IEM_WITH_SETJMP */
376
377	/**
378	* Fetches the next opcode byte, longjmp on error.
379	*
380	* @returns The opcode byte.
381	* @param pVCpu The cross context virtual CPU structure of the calling thread.
382	*/
383	DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
384	{
385	# ifdef IEM_WITH_CODE_TLB
386	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
387	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
388	if (RT_LIKELY( pbBuf != NULL
389	&& offBuf < pVCpu->iem.s.cbInstrBuf))
390	{
391	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
392	return pbBuf[offBuf];
393	}
394	# else
395	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
396	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
397	{
398	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
399	return pVCpu->iem.s.abOpcode[offOpcode];
400	}
401	# endif
402	return iemOpcodeGetNextU8SlowJmp(pVCpu);
403	}
404
405	#endif /* IEM_WITH_SETJMP */
406
407	/**
408	* Fetches the next opcode byte, returns automatically on failure.
409	*
410	* @param a_pu8 Where to return the opcode byte.
411	* @remark Implicitly references pVCpu.
412	*/
413	#ifndef IEM_WITH_SETJMP
414	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
415	do \
416	{ \
417	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
418	if (rcStrict2 == VINF_SUCCESS) \
419	{ /* likely */ } \
420	else \
421	return rcStrict2; \
422	} while (0)
423	#else
424	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
425	#endif /* IEM_WITH_SETJMP */
426
427
428	#ifndef IEM_WITH_SETJMP
429	/**
430	* Fetches the next signed byte from the opcode stream.
431	*
432	* @returns Strict VBox status code.
433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
434	* @param pi8 Where to return the signed byte.
435	*/
436	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
437	{
438	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
439	}
440	#endif /* !IEM_WITH_SETJMP */
441
442
443	/**
444	* Fetches the next signed byte from the opcode stream, returning automatically
445	* on failure.
446	*
447	* @param a_pi8 Where to return the signed byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	#ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
455	if (rcStrict2 != VINF_SUCCESS) \
456	return rcStrict2; \
457	} while (0)
458	#else /* IEM_WITH_SETJMP */
459	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
460
461	#endif /* IEM_WITH_SETJMP */
462
463
464	#ifndef IEM_WITH_SETJMP
465	/**
466	* Fetches the next signed byte from the opcode stream, extending it to
467	* unsigned 16-bit.
468	*
469	* @returns Strict VBox status code.
470	* @param pVCpu The cross context virtual CPU structure of the calling thread.
471	* @param pu16 Where to return the unsigned word.
472	*/
473	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
474	{
475	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
476	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
477	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
478
479	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
480	pVCpu->iem.s.offOpcode = offOpcode + 1;
481	return VINF_SUCCESS;
482	}
483	#endif /* !IEM_WITH_SETJMP */
484
485	/**
486	* Fetches the next signed byte from the opcode stream and sign-extending it to
487	* a word, returning automatically on failure.
488	*
489	* @param a_pu16 Where to return the word.
490	* @remark Implicitly references pVCpu.
491	*/
492	#ifndef IEM_WITH_SETJMP
493	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
494	do \
495	{ \
496	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
497	if (rcStrict2 != VINF_SUCCESS) \
498	return rcStrict2; \
499	} while (0)
500	#else
501	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
502	#endif
503
504	#ifndef IEM_WITH_SETJMP
505	/**
506	* Fetches the next signed byte from the opcode stream, extending it to
507	* unsigned 32-bit.
508	*
509	* @returns Strict VBox status code.
510	* @param pVCpu The cross context virtual CPU structure of the calling thread.
511	* @param pu32 Where to return the unsigned dword.
512	*/
513	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
514	{
515	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
516	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
517	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
518
519	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
520	pVCpu->iem.s.offOpcode = offOpcode + 1;
521	return VINF_SUCCESS;
522	}
523	#endif /* !IEM_WITH_SETJMP */
524
525	/**
526	* Fetches the next signed byte from the opcode stream and sign-extending it to
527	* a word, returning automatically on failure.
528	*
529	* @param a_pu32 Where to return the word.
530	* @remark Implicitly references pVCpu.
531	*/
532	#ifndef IEM_WITH_SETJMP
533	#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
534	do \
535	{ \
536	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
537	if (rcStrict2 != VINF_SUCCESS) \
538	return rcStrict2; \
539	} while (0)
540	#else
541	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
542	#endif
543
544
545	#ifndef IEM_WITH_SETJMP
546	/**
547	* Fetches the next signed byte from the opcode stream, extending it to
548	* unsigned 64-bit.
549	*
550	* @returns Strict VBox status code.
551	* @param pVCpu The cross context virtual CPU structure of the calling thread.
552	* @param pu64 Where to return the unsigned qword.
553	*/
554	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
555	{
556	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
557	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
558	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
559
560	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
561	pVCpu->iem.s.offOpcode = offOpcode + 1;
562	return VINF_SUCCESS;
563	}
564	#endif /* !IEM_WITH_SETJMP */
565
566	/**
567	* Fetches the next signed byte from the opcode stream and sign-extending it to
568	* a word, returning automatically on failure.
569	*
570	* @param a_pu64 Where to return the word.
571	* @remark Implicitly references pVCpu.
572	*/
573	#ifndef IEM_WITH_SETJMP
574	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
575	do \
576	{ \
577	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
578	if (rcStrict2 != VINF_SUCCESS) \
579	return rcStrict2; \
580	} while (0)
581	#else
582	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
583	#endif
584
585
586	#ifndef IEM_WITH_SETJMP
587	/**
588	* Fetches the next opcode byte.
589	*
590	* @returns Strict VBox status code.
591	* @param pVCpu The cross context virtual CPU structure of the
592	* calling thread.
593	* @param pu8 Where to return the opcode byte.
594	*/
595	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
596	{
597	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
598	pVCpu->iem.s.offModRm = offOpcode;
599	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
600	{
601	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
602	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
603	return VINF_SUCCESS;
604	}
605	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
606	}
607	#else /* IEM_WITH_SETJMP */
608	/**
609	* Fetches the next opcode byte, longjmp on error.
610	*
611	* @returns The opcode byte.
612	* @param pVCpu The cross context virtual CPU structure of the calling thread.
613	*/
614	DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
615	{
616	# ifdef IEM_WITH_CODE_TLB
617	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
618	pVCpu->iem.s.offModRm = offBuf;
619	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
620	if (RT_LIKELY( pbBuf != NULL
621	&& offBuf < pVCpu->iem.s.cbInstrBuf))
622	{
623	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
624	return pbBuf[offBuf];
625	}
626	# else
627	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
628	pVCpu->iem.s.offModRm = offOpcode;
629	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
630	{
631	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
632	return pVCpu->iem.s.abOpcode[offOpcode];
633	}
634	# endif
635	return iemOpcodeGetNextU8SlowJmp(pVCpu);
636	}
637	#endif /* IEM_WITH_SETJMP */
638
639	/**
640	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
641	* on failure.
642	*
643	* Will note down the position of the ModR/M byte for VT-x exits.
644	*
645	* @param a_pbRm Where to return the RM opcode byte.
646	* @remark Implicitly references pVCpu.
647	*/
648	#ifndef IEM_WITH_SETJMP
649	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
650	do \
651	{ \
652	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
653	if (rcStrict2 == VINF_SUCCESS) \
654	{ /* likely */ } \
655	else \
656	return rcStrict2; \
657	} while (0)
658	#else
659	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
660	#endif /* IEM_WITH_SETJMP */
661
662
663	#ifndef IEM_WITH_SETJMP
664
665	/**
666	* Fetches the next opcode word.
667	*
668	* @returns Strict VBox status code.
669	* @param pVCpu The cross context virtual CPU structure of the calling thread.
670	* @param pu16 Where to return the opcode word.
671	*/
672	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
673	{
674	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
675	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
676	{
677	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
678	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
679	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
680	# else
681	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
682	# endif
683	return VINF_SUCCESS;
684	}
685	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
686	}
687
688	#else /* IEM_WITH_SETJMP */
689
690	/**
691	* Fetches the next opcode word, longjmp on error.
692	*
693	* @returns The opcode word.
694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
695	*/
696	DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
697	{
698	# ifdef IEM_WITH_CODE_TLB
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
705	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
706	return (uint16_t const )&pbBuf[offBuf];
707	# else
708	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
709	# endif
710	}
711	# else
712	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
713	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
714	{
715	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
716	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
717	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
718	# else
719	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
720	# endif
721	}
722	# endif
723	return iemOpcodeGetNextU16SlowJmp(pVCpu);
724	}
725
726	#endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode word, returns automatically on failure.
730	*
731	* @param a_pu16 Where to return the opcode word.
732	* @remark Implicitly references pVCpu.
733	*/
734	#ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
739	if (rcStrict2 != VINF_SUCCESS) \
740	return rcStrict2; \
741	} while (0)
742	#else
743	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
744	#endif
745
746	#ifndef IEM_WITH_SETJMP
747	/**
748	* Fetches the next opcode word, zero extending it to a double word.
749	*
750	* @returns Strict VBox status code.
751	* @param pVCpu The cross context virtual CPU structure of the calling thread.
752	* @param pu32 Where to return the opcode double word.
753	*/
754	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
755	{
756	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
757	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
758	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
759
760	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
761	pVCpu->iem.s.offOpcode = offOpcode + 2;
762	return VINF_SUCCESS;
763	}
764	#endif /* !IEM_WITH_SETJMP */
765
766	/**
767	* Fetches the next opcode word and zero extends it to a double word, returns
768	* automatically on failure.
769	*
770	* @param a_pu32 Where to return the opcode double word.
771	* @remark Implicitly references pVCpu.
772	*/
773	#ifndef IEM_WITH_SETJMP
774	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
775	do \
776	{ \
777	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
778	if (rcStrict2 != VINF_SUCCESS) \
779	return rcStrict2; \
780	} while (0)
781	#else
782	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
783	#endif
784
785	#ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next opcode word, zero extending it to a quad word.
788	*
789	* @returns Strict VBox status code.
790	* @param pVCpu The cross context virtual CPU structure of the calling thread.
791	* @param pu64 Where to return the opcode quad word.
792	*/
793	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
794	{
795	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
796	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
797	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
798
799	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
800	pVCpu->iem.s.offOpcode = offOpcode + 2;
801	return VINF_SUCCESS;
802	}
803	#endif /* !IEM_WITH_SETJMP */
804
805	/**
806	* Fetches the next opcode word and zero extends it to a quad word, returns
807	* automatically on failure.
808	*
809	* @param a_pu64 Where to return the opcode quad word.
810	* @remark Implicitly references pVCpu.
811	*/
812	#ifndef IEM_WITH_SETJMP
813	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
814	do \
815	{ \
816	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
817	if (rcStrict2 != VINF_SUCCESS) \
818	return rcStrict2; \
819	} while (0)
820	#else
821	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
822	#endif
823
824
825	#ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed word from the opcode stream.
828	*
829	* @returns Strict VBox status code.
830	* @param pVCpu The cross context virtual CPU structure of the calling thread.
831	* @param pi16 Where to return the signed word.
832	*/
833	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
834	{
835	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
836	}
837	#endif /* !IEM_WITH_SETJMP */
838
839
840	/**
841	* Fetches the next signed word from the opcode stream, returning automatically
842	* on failure.
843	*
844	* @param a_pi16 Where to return the signed word.
845	* @remark Implicitly references pVCpu.
846	*/
847	#ifndef IEM_WITH_SETJMP
848	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
849	do \
850	{ \
851	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
852	if (rcStrict2 != VINF_SUCCESS) \
853	return rcStrict2; \
854	} while (0)
855	#else
856	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
857	#endif
858
859	#ifndef IEM_WITH_SETJMP
860
861	/**
862	* Fetches the next opcode dword.
863	*
864	* @returns Strict VBox status code.
865	* @param pVCpu The cross context virtual CPU structure of the calling thread.
866	* @param pu32 Where to return the opcode double word.
867	*/
868	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
869	{
870	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
871	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
872	{
873	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
874	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
875	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
876	# else
877	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
878	pVCpu->iem.s.abOpcode[offOpcode + 1],
879	pVCpu->iem.s.abOpcode[offOpcode + 2],
880	pVCpu->iem.s.abOpcode[offOpcode + 3]);
881	# endif
882	return VINF_SUCCESS;
883	}
884	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
885	}
886
887	#else /* IEM_WITH_SETJMP */
888
889	/**
890	* Fetches the next opcode dword, longjmp on error.
891	*
892	* @returns The opcode dword.
893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
894	*/
895	DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
896	{
897	# ifdef IEM_WITH_CODE_TLB
898	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
899	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
900	if (RT_LIKELY( pbBuf != NULL
901	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
902	{
903	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
904	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
905	return (uint32_t const )&pbBuf[offBuf];
906	# else
907	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
908	pbBuf[offBuf + 1],
909	pbBuf[offBuf + 2],
910	pbBuf[offBuf + 3]);
911	# endif
912	}
913	# else
914	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
915	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
916	{
917	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
918	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
919	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
920	# else
921	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
922	pVCpu->iem.s.abOpcode[offOpcode + 1],
923	pVCpu->iem.s.abOpcode[offOpcode + 2],
924	pVCpu->iem.s.abOpcode[offOpcode + 3]);
925	# endif
926	}
927	# endif
928	return iemOpcodeGetNextU32SlowJmp(pVCpu);
929	}
930
931	#endif /* IEM_WITH_SETJMP */
932
933	/**
934	* Fetches the next opcode dword, returns automatically on failure.
935	*
936	* @param a_pu32 Where to return the opcode dword.
937	* @remark Implicitly references pVCpu.
938	*/
939	#ifndef IEM_WITH_SETJMP
940	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
941	do \
942	{ \
943	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
944	if (rcStrict2 != VINF_SUCCESS) \
945	return rcStrict2; \
946	} while (0)
947	#else
948	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
949	#endif
950
951	#ifndef IEM_WITH_SETJMP
952	/**
953	* Fetches the next opcode dword, zero extending it to a quad word.
954	*
955	* @returns Strict VBox status code.
956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
957	* @param pu64 Where to return the opcode quad word.
958	*/
959	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
960	{
961	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
962	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
963	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
964
965	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
966	pVCpu->iem.s.abOpcode[offOpcode + 1],
967	pVCpu->iem.s.abOpcode[offOpcode + 2],
968	pVCpu->iem.s.abOpcode[offOpcode + 3]);
969	pVCpu->iem.s.offOpcode = offOpcode + 4;
970	return VINF_SUCCESS;
971	}
972	#endif /* !IEM_WITH_SETJMP */
973
974	/**
975	* Fetches the next opcode dword and zero extends it to a quad word, returns
976	* automatically on failure.
977	*
978	* @param a_pu64 Where to return the opcode quad word.
979	* @remark Implicitly references pVCpu.
980	*/
981	#ifndef IEM_WITH_SETJMP
982	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
983	do \
984	{ \
985	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
986	if (rcStrict2 != VINF_SUCCESS) \
987	return rcStrict2; \
988	} while (0)
989	#else
990	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
991	#endif
992
993
994	#ifndef IEM_WITH_SETJMP
995	/**
996	* Fetches the next signed double word from the opcode stream.
997	*
998	* @returns Strict VBox status code.
999	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1000	* @param pi32 Where to return the signed double word.
1001	*/
1002	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
1003	{
1004	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1005	}
1006	#endif
1007
1008	/**
1009	* Fetches the next signed double word from the opcode stream, returning
1010	* automatically on failure.
1011	*
1012	* @param a_pi32 Where to return the signed double word.
1013	* @remark Implicitly references pVCpu.
1014	*/
1015	#ifndef IEM_WITH_SETJMP
1016	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1017	do \
1018	{ \
1019	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1020	if (rcStrict2 != VINF_SUCCESS) \
1021	return rcStrict2; \
1022	} while (0)
1023	#else
1024	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1025	#endif
1026
1027	#ifndef IEM_WITH_SETJMP
1028	/**
1029	* Fetches the next opcode dword, sign extending it into a quad word.
1030	*
1031	* @returns Strict VBox status code.
1032	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1033	* @param pu64 Where to return the opcode quad word.
1034	*/
1035	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
1036	{
1037	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1038	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1039	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1040
1041	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1042	pVCpu->iem.s.abOpcode[offOpcode + 1],
1043	pVCpu->iem.s.abOpcode[offOpcode + 2],
1044	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1045	*pu64 = i32;
1046	pVCpu->iem.s.offOpcode = offOpcode + 4;
1047	return VINF_SUCCESS;
1048	}
1049	#endif /* !IEM_WITH_SETJMP */
1050
1051	/**
1052	* Fetches the next opcode double word and sign extends it to a quad word,
1053	* returns automatically on failure.
1054	*
1055	* @param a_pu64 Where to return the opcode quad word.
1056	* @remark Implicitly references pVCpu.
1057	*/
1058	#ifndef IEM_WITH_SETJMP
1059	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1060	do \
1061	{ \
1062	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1063	if (rcStrict2 != VINF_SUCCESS) \
1064	return rcStrict2; \
1065	} while (0)
1066	#else
1067	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1068	#endif
1069
1070	#ifndef IEM_WITH_SETJMP
1071
1072	/**
1073	* Fetches the next opcode qword.
1074	*
1075	* @returns Strict VBox status code.
1076	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1077	* @param pu64 Where to return the opcode qword.
1078	*/
1079	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
1080	{
1081	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1082	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1083	{
1084	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1085	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1086	# else
1087	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1088	pVCpu->iem.s.abOpcode[offOpcode + 1],
1089	pVCpu->iem.s.abOpcode[offOpcode + 2],
1090	pVCpu->iem.s.abOpcode[offOpcode + 3],
1091	pVCpu->iem.s.abOpcode[offOpcode + 4],
1092	pVCpu->iem.s.abOpcode[offOpcode + 5],
1093	pVCpu->iem.s.abOpcode[offOpcode + 6],
1094	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1095	# endif
1096	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1097	return VINF_SUCCESS;
1098	}
1099	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1100	}
1101
1102	#else /* IEM_WITH_SETJMP */
1103
1104	/**
1105	* Fetches the next opcode qword, longjmp on error.
1106	*
1107	* @returns The opcode qword.
1108	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1109	*/
1110	DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
1111	{
1112	# ifdef IEM_WITH_CODE_TLB
1113	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1114	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1115	if (RT_LIKELY( pbBuf != NULL
1116	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1117	{
1118	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1119	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1120	return (uint64_t const )&pbBuf[offBuf];
1121	# else
1122	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1123	pbBuf[offBuf + 1],
1124	pbBuf[offBuf + 2],
1125	pbBuf[offBuf + 3],
1126	pbBuf[offBuf + 4],
1127	pbBuf[offBuf + 5],
1128	pbBuf[offBuf + 6],
1129	pbBuf[offBuf + 7]);
1130	# endif
1131	}
1132	# else
1133	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1134	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1135	{
1136	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1137	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1138	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1139	# else
1140	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1141	pVCpu->iem.s.abOpcode[offOpcode + 1],
1142	pVCpu->iem.s.abOpcode[offOpcode + 2],
1143	pVCpu->iem.s.abOpcode[offOpcode + 3],
1144	pVCpu->iem.s.abOpcode[offOpcode + 4],
1145	pVCpu->iem.s.abOpcode[offOpcode + 5],
1146	pVCpu->iem.s.abOpcode[offOpcode + 6],
1147	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1148	# endif
1149	}
1150	# endif
1151	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1152	}
1153
1154	#endif /* IEM_WITH_SETJMP */
1155
1156	/**
1157	* Fetches the next opcode quad word, returns automatically on failure.
1158	*
1159	* @param a_pu64 Where to return the opcode quad word.
1160	* @remark Implicitly references pVCpu.
1161	*/
1162	#ifndef IEM_WITH_SETJMP
1163	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1164	do \
1165	{ \
1166	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1167	if (rcStrict2 != VINF_SUCCESS) \
1168	return rcStrict2; \
1169	} while (0)
1170	#else
1171	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1172	#endif
1173
1174
1175	/** @name Misc Worker Functions.
1176	* @{
1177	*/
1178
1179	/**
1180	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1181	* not (kind of obsolete now).
1182	*
1183	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1184	*/
1185	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1186
1187	/**
1188	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1189	*
1190	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1191	* @param a_fEfl The new EFLAGS.
1192	*/
1193	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1194
1195
1196	/**
1197	* Loads a NULL data selector into a selector register, both the hidden and
1198	* visible parts, in protected mode.
1199	*
1200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1201	* @param pSReg Pointer to the segment register.
1202	* @param uRpl The RPL.
1203	*/
1204	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
1205	{
1206	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1207	* data selector in protected mode. */
1208	pSReg->Sel = uRpl;
1209	pSReg->ValidSel = uRpl;
1210	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1211	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1212	{
1213	/* VT-x (Intel 3960x) observed doing something like this. */
1214	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1215	pSReg->u32Limit = UINT32_MAX;
1216	pSReg->u64Base = 0;
1217	}
1218	else
1219	{
1220	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1221	pSReg->u32Limit = 0;
1222	pSReg->u64Base = 0;
1223	}
1224	}
1225
1226	/** @} */
1227
1228
1229	/*
1230	*
1231	* Helpers routines.
1232	* Helpers routines.
1233	* Helpers routines.
1234	*
1235	*/
1236
1237	/**
1238	* Recalculates the effective operand size.
1239	*
1240	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1241	*/
1242	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu)
1243	{
1244	switch (pVCpu->iem.s.enmCpuMode)
1245	{
1246	case IEMMODE_16BIT:
1247	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1248	break;
1249	case IEMMODE_32BIT:
1250	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1251	break;
1252	case IEMMODE_64BIT:
1253	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1254	{
1255	case 0:
1256	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1257	break;
1258	case IEM_OP_PRF_SIZE_OP:
1259	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1260	break;
1261	case IEM_OP_PRF_SIZE_REX_W:
1262	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1263	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1264	break;
1265	}
1266	break;
1267	default:
1268	AssertFailed();
1269	}
1270	}
1271
1272
1273	/**
1274	* Sets the default operand size to 64-bit and recalculates the effective
1275	* operand size.
1276	*
1277	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1278	*/
1279	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
1280	{
1281	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1282	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1283	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1284	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1285	else
1286	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1287	}
1288
1289
1290
1291
1292	/** @name Register Access.
1293	* @{
1294	*/
1295
1296	/**
1297	* Gets a reference (pointer) to the specified hidden segment register.
1298	*
1299	* @returns Hidden register reference.
1300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1301	* @param iSegReg The segment register.
1302	*/
1303	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
1304	{
1305	Assert(iSegReg < X86_SREG_COUNT);
1306	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1307	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1308
1309	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1310	return pSReg;
1311	}
1312
1313
1314	/**
1315	* Ensures that the given hidden segment register is up to date.
1316	*
1317	* @returns Hidden register reference.
1318	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1319	* @param pSReg The segment register.
1320	*/
1321	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1322	{
1323	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1324	NOREF(pVCpu);
1325	return pSReg;
1326	}
1327
1328
1329	/**
1330	* Gets a reference (pointer) to the specified segment register (the selector
1331	* value).
1332	*
1333	* @returns Pointer to the selector variable.
1334	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1335	* @param iSegReg The segment register.
1336	*/
1337	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
1338	{
1339	Assert(iSegReg < X86_SREG_COUNT);
1340	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1341	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1342	}
1343
1344
1345	/**
1346	* Fetches the selector value of a segment register.
1347	*
1348	* @returns The selector value.
1349	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1350	* @param iSegReg The segment register.
1351	*/
1352	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
1353	{
1354	Assert(iSegReg < X86_SREG_COUNT);
1355	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1356	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1357	}
1358
1359
1360	/**
1361	* Fetches the base address value of a segment register.
1362	*
1363	* @returns The selector value.
1364	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1365	* @param iSegReg The segment register.
1366	*/
1367	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1368	{
1369	Assert(iSegReg < X86_SREG_COUNT);
1370	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1371	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1372	}
1373
1374
1375	/**
1376	* Gets a reference (pointer) to the specified general purpose register.
1377	*
1378	* @returns Register reference.
1379	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1380	* @param iReg The general purpose register.
1381	*/
1382	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
1383	{
1384	Assert(iReg < 16);
1385	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1386	}
1387
1388
1389	/**
1390	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1391	*
1392	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1393	*
1394	* @returns Register reference.
1395	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1396	* @param iReg The register.
1397	*/
1398	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
1399	{
1400	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1401	{
1402	Assert(iReg < 16);
1403	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1404	}
1405	/* high 8-bit register. */
1406	Assert(iReg < 8);
1407	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1408	}
1409
1410
1411	/**
1412	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1413	*
1414	* @returns Register reference.
1415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1416	* @param iReg The register.
1417	*/
1418	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
1419	{
1420	Assert(iReg < 16);
1421	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1422	}
1423
1424
1425	/**
1426	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1427	*
1428	* @returns Register reference.
1429	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1430	* @param iReg The register.
1431	*/
1432	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
1433	{
1434	Assert(iReg < 16);
1435	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1436	}
1437
1438
1439	/**
1440	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1441	*
1442	* @returns Register reference.
1443	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1444	* @param iReg The register.
1445	*/
1446	DECLINLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg)
1447	{
1448	Assert(iReg < 16);
1449	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1450	}
1451
1452
1453	/**
1454	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1455	*
1456	* @returns Register reference.
1457	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1458	* @param iReg The register.
1459	*/
1460	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
1461	{
1462	Assert(iReg < 64);
1463	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1464	}
1465
1466
1467	/**
1468	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1469	*
1470	* @returns Register reference.
1471	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1472	* @param iReg The register.
1473	*/
1474	DECLINLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg)
1475	{
1476	Assert(iReg < 16);
1477	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1478	}
1479
1480
1481	/**
1482	* Gets a reference (pointer) to the specified segment register's base address.
1483	*
1484	* @returns Segment register base address reference.
1485	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1486	* @param iSegReg The segment selector.
1487	*/
1488	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1489	{
1490	Assert(iSegReg < X86_SREG_COUNT);
1491	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1492	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1493	}
1494
1495
1496	/**
1497	* Fetches the value of a 8-bit general purpose register.
1498	*
1499	* @returns The register value.
1500	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1501	* @param iReg The register.
1502	*/
1503	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
1504	{
1505	return *iemGRegRefU8(pVCpu, iReg);
1506	}
1507
1508
1509	/**
1510	* Fetches the value of a 16-bit general purpose register.
1511	*
1512	* @returns The register value.
1513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1514	* @param iReg The register.
1515	*/
1516	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
1517	{
1518	Assert(iReg < 16);
1519	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1520	}
1521
1522
1523	/**
1524	* Fetches the value of a 32-bit general purpose register.
1525	*
1526	* @returns The register value.
1527	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1528	* @param iReg The register.
1529	*/
1530	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
1531	{
1532	Assert(iReg < 16);
1533	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1534	}
1535
1536
1537	/**
1538	* Fetches the value of a 64-bit general purpose register.
1539	*
1540	* @returns The register value.
1541	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1542	* @param iReg The register.
1543	*/
1544	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
1545	{
1546	Assert(iReg < 16);
1547	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1548	}
1549
1550
1551	/**
1552	* Get the address of the top of the stack.
1553	*
1554	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1555	*/
1556	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
1557	{
1558	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1559	return pVCpu->cpum.GstCtx.rsp;
1560	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1561	return pVCpu->cpum.GstCtx.esp;
1562	return pVCpu->cpum.GstCtx.sp;
1563	}
1564
1565
1566	/**
1567	* Updates the RIP/EIP/IP to point to the next instruction.
1568	*
1569	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1570	* @param cbInstr The number of bytes to add.
1571	*/
1572	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr)
1573	{
1574	/*
1575	* Advance RIP.
1576	*
1577	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
1578	* while in all other modes except LM64 the updates are 32-bit. This means
1579	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
1580	* 4GB and 64KB rollovers, and decide whether anything needs masking.
1581	*
1582	* See PC wrap around tests in bs3-cpu-weird-1.
1583	*/
1584	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
1585	uint64_t const uRipNext = uRipPrev + cbInstr;
1586	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
1587	\|\| CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
1588	pVCpu->cpum.GstCtx.rip = uRipNext;
1589	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
1590	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
1591	else
1592	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
1593	}
1594
1595
1596	/**
1597	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
1598	* following EFLAGS bits are set:
1599	* - X86_EFL_RF - clear it.
1600	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
1601	* - X86_EFL_TF - generate single step \#DB trap.
1602	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
1603	* instruction).
1604	*
1605	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
1606	* a #DB due to TF (single stepping) or a DRx non-instruction breakpoint
1607	* takes priority over both NMIs and hardware interrupts. So, neither is
1608	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
1609	* either unsupported will be triggered on-top of any \#DB raised here.)
1610	*
1611	* The RF flag only needs to be cleared here as it only suppresses instruction
1612	* breakpoints which are not raised here (happens synchronously during
1613	* instruction fetching).
1614	*
1615	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
1616	* status has no bearing on whether \#DB exceptions are raised.
1617	*
1618	* @note This must NOT be called by the two instructions setting the
1619	* CPUMCTX_INHIBIT_SHADOW_SS flag.
1620	*
1621	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
1622	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
1623	* Stacks}
1624	*/
1625	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu)
1626	{
1627	/*
1628	* Normally we're just here to clear RF and/or interrupt shadow bits.
1629	*/
1630	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK)) == 0))
1631	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
1632	else
1633	{
1634	#if 1
1635	/*
1636	* Raise a #DB.
1637	*/
1638	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1639	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
1640	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
1641	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1642	/** @todo Do we set all pending \#DB events, or just one? */
1643	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
1644	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1645	pVCpu->cpum.GstCtx.rflags.uBoth));
1646	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1647	return iemRaiseDebugException(pVCpu);
1648	#else
1649	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1650	#endif
1651	}
1652	return VINF_SUCCESS;
1653	}
1654
1655
1656	/**
1657	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
1658	*
1659	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1660	*/
1661	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu)
1662	{
1663	/*
1664	* We assume that most of the time nothing actually needs doing here.
1665	*/
1666	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1667	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
1668	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK)) ))
1669	return VINF_SUCCESS;
1670	return iemFinishInstructionWithFlagsSet(pVCpu);
1671	}
1672
1673
1674	/**
1675	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
1676	* and CPUMCTX_INHIBIT_SHADOW.
1677	*
1678	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1679	* @param cbInstr The number of bytes to add.
1680	*/
1681	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1682	{
1683	iemRegAddToRip(pVCpu, cbInstr);
1684	return iemRegFinishClearingRF(pVCpu);
1685	}
1686
1687
1688	/**
1689	* Extended version of iemFinishInstructionWithFlagsSet that goes with
1690	* iemRegAddToRipAndFinishingClearingRfEx.
1691	*
1692	* See iemFinishInstructionWithFlagsSet() for details.
1693	*/
1694	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu)
1695	{
1696	/*
1697	* Raise a #DB.
1698	*/
1699	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1700	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
1701	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
1702	/** @todo Do we set all pending \#DB events, or just one? */
1703	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
1704	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1705	pVCpu->cpum.GstCtx.rflags.uBoth));
1706	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
1707	return iemRaiseDebugException(pVCpu);
1708	}
1709
1710
1711	/**
1712	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
1713	* others potentially updating EFLAGS.TF.
1714	*
1715	* The single step event must be generated using the TF value at the start of
1716	* the instruction, not the new value set by it.
1717	*
1718	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1719	* @param cbInstr The number of bytes to add.
1720	* @param fEflOld The EFLAGS at the start of the instruction
1721	* execution.
1722	*/
1723	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld)
1724	{
1725	iemRegAddToRip(pVCpu, cbInstr);
1726	if (!(fEflOld & X86_EFL_TF))
1727	return iemRegFinishClearingRF(pVCpu);
1728	return iemFinishInstructionWithTfSet(pVCpu);
1729	}
1730
1731
1732	/**
1733	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1734	*
1735	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1736	*/
1737	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu)
1738	{
1739	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1740	}
1741
1742
1743	/**
1744	* Adds to the stack pointer.
1745	*
1746	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1747	* @param cbToAdd The number of bytes to add (8-bit!).
1748	*/
1749	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
1750	{
1751	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1752	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1753	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1754	pVCpu->cpum.GstCtx.esp += cbToAdd;
1755	else
1756	pVCpu->cpum.GstCtx.sp += cbToAdd;
1757	}
1758
1759
1760	/**
1761	* Subtracts from the stack pointer.
1762	*
1763	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1764	* @param cbToSub The number of bytes to subtract (8-bit!).
1765	*/
1766	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
1767	{
1768	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1769	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1770	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1771	pVCpu->cpum.GstCtx.esp -= cbToSub;
1772	else
1773	pVCpu->cpum.GstCtx.sp -= cbToSub;
1774	}
1775
1776
1777	/**
1778	* Adds to the temporary stack pointer.
1779	*
1780	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1781	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1782	* @param cbToAdd The number of bytes to add (16-bit).
1783	*/
1784	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
1785	{
1786	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1787	pTmpRsp->u += cbToAdd;
1788	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1789	pTmpRsp->DWords.dw0 += cbToAdd;
1790	else
1791	pTmpRsp->Words.w0 += cbToAdd;
1792	}
1793
1794
1795	/**
1796	* Subtracts from the temporary stack pointer.
1797	*
1798	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1799	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1800	* @param cbToSub The number of bytes to subtract.
1801	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1802	* expecting that.
1803	*/
1804	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
1805	{
1806	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1807	pTmpRsp->u -= cbToSub;
1808	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1809	pTmpRsp->DWords.dw0 -= cbToSub;
1810	else
1811	pTmpRsp->Words.w0 -= cbToSub;
1812	}
1813
1814
1815	/**
1816	* Calculates the effective stack address for a push of the specified size as
1817	* well as the new RSP value (upper bits may be masked).
1818	*
1819	* @returns Effective stack addressf for the push.
1820	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1821	* @param cbItem The size of the stack item to pop.
1822	* @param puNewRsp Where to return the new RSP value.
1823	*/
1824	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1825	{
1826	RTUINT64U uTmpRsp;
1827	RTGCPTR GCPtrTop;
1828	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1829
1830	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1831	GCPtrTop = uTmpRsp.u -= cbItem;
1832	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1833	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
1834	else
1835	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
1836	*puNewRsp = uTmpRsp.u;
1837	return GCPtrTop;
1838	}
1839
1840
1841	/**
1842	* Gets the current stack pointer and calculates the value after a pop of the
1843	* specified size.
1844	*
1845	* @returns Current stack pointer.
1846	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1847	* @param cbItem The size of the stack item to pop.
1848	* @param puNewRsp Where to return the new RSP value.
1849	*/
1850	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1851	{
1852	RTUINT64U uTmpRsp;
1853	RTGCPTR GCPtrTop;
1854	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1855
1856	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1857	{
1858	GCPtrTop = uTmpRsp.u;
1859	uTmpRsp.u += cbItem;
1860	}
1861	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1862	{
1863	GCPtrTop = uTmpRsp.DWords.dw0;
1864	uTmpRsp.DWords.dw0 += cbItem;
1865	}
1866	else
1867	{
1868	GCPtrTop = uTmpRsp.Words.w0;
1869	uTmpRsp.Words.w0 += cbItem;
1870	}
1871	*puNewRsp = uTmpRsp.u;
1872	return GCPtrTop;
1873	}
1874
1875
1876	/**
1877	* Calculates the effective stack address for a push of the specified size as
1878	* well as the new temporary RSP value (upper bits may be masked).
1879	*
1880	* @returns Effective stack addressf for the push.
1881	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1882	* @param pTmpRsp The temporary stack pointer. This is updated.
1883	* @param cbItem The size of the stack item to pop.
1884	*/
1885	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1886	{
1887	RTGCPTR GCPtrTop;
1888
1889	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1890	GCPtrTop = pTmpRsp->u -= cbItem;
1891	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1892	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
1893	else
1894	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
1895	return GCPtrTop;
1896	}
1897
1898
1899	/**
1900	* Gets the effective stack address for a pop of the specified size and
1901	* calculates and updates the temporary RSP.
1902	*
1903	* @returns Current stack pointer.
1904	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1905	* @param pTmpRsp The temporary stack pointer. This is updated.
1906	* @param cbItem The size of the stack item to pop.
1907	*/
1908	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1909	{
1910	RTGCPTR GCPtrTop;
1911	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1912	{
1913	GCPtrTop = pTmpRsp->u;
1914	pTmpRsp->u += cbItem;
1915	}
1916	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1917	{
1918	GCPtrTop = pTmpRsp->DWords.dw0;
1919	pTmpRsp->DWords.dw0 += cbItem;
1920	}
1921	else
1922	{
1923	GCPtrTop = pTmpRsp->Words.w0;
1924	pTmpRsp->Words.w0 += cbItem;
1925	}
1926	return GCPtrTop;
1927	}
1928
1929	/** @} */
1930
1931
1932	/** @name FPU access and helpers.
1933	*
1934	* @{
1935	*/
1936
1937
1938	/**
1939	* Hook for preparing to use the host FPU.
1940	*
1941	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1942	*
1943	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1944	*/
1945	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
1946	{
1947	#ifdef IN_RING3
1948	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1949	#else
1950	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
1951	#endif
1952	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1953	}
1954
1955
1956	/**
1957	* Hook for preparing to use the host FPU for SSE.
1958	*
1959	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1960	*
1961	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1962	*/
1963	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
1964	{
1965	iemFpuPrepareUsage(pVCpu);
1966	}
1967
1968
1969	/**
1970	* Hook for preparing to use the host FPU for AVX.
1971	*
1972	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1973	*
1974	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1975	*/
1976	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
1977	{
1978	iemFpuPrepareUsage(pVCpu);
1979	}
1980
1981
1982	/**
1983	* Hook for actualizing the guest FPU state before the interpreter reads it.
1984	*
1985	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1986	*
1987	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1988	*/
1989	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
1990	{
1991	#ifdef IN_RING3
1992	NOREF(pVCpu);
1993	#else
1994	CPUMRZFpuStateActualizeForRead(pVCpu);
1995	#endif
1996	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1997	}
1998
1999
2000	/**
2001	* Hook for actualizing the guest FPU state before the interpreter changes it.
2002	*
2003	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2004	*
2005	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2006	*/
2007	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
2008	{
2009	#ifdef IN_RING3
2010	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2011	#else
2012	CPUMRZFpuStateActualizeForChange(pVCpu);
2013	#endif
2014	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2015	}
2016
2017
2018	/**
2019	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2020	* only.
2021	*
2022	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2023	*
2024	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2025	*/
2026	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
2027	{
2028	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2029	NOREF(pVCpu);
2030	#else
2031	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2032	#endif
2033	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2034	}
2035
2036
2037	/**
2038	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2039	* read+write.
2040	*
2041	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2042	*
2043	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2044	*/
2045	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
2046	{
2047	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2048	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2049	#else
2050	CPUMRZFpuStateActualizeForChange(pVCpu);
2051	#endif
2052	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2053
2054	/* Make sure any changes are loaded the next time around. */
2055	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2056	}
2057
2058
2059	/**
2060	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2061	* only.
2062	*
2063	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2064	*
2065	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2066	*/
2067	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
2068	{
2069	#ifdef IN_RING3
2070	NOREF(pVCpu);
2071	#else
2072	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2073	#endif
2074	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2075	}
2076
2077
2078	/**
2079	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2080	* read+write.
2081	*
2082	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2083	*
2084	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2085	*/
2086	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
2087	{
2088	#ifdef IN_RING3
2089	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2090	#else
2091	CPUMRZFpuStateActualizeForChange(pVCpu);
2092	#endif
2093	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2094
2095	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2096	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2097	}
2098
2099
2100	/**
2101	* Stores a QNaN value into a FPU register.
2102	*
2103	* @param pReg Pointer to the register.
2104	*/
2105	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
2106	{
2107	pReg->au32[0] = UINT32_C(0x00000000);
2108	pReg->au32[1] = UINT32_C(0xc0000000);
2109	pReg->au16[4] = UINT16_C(0xffff);
2110	}
2111
2112
2113	/**
2114	* Updates the FOP, FPU.CS and FPUIP registers.
2115	*
2116	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2117	* @param pFpuCtx The FPU context.
2118	*/
2119	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
2120	{
2121	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
2122	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
2123	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2124	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2125	{
2126	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2127	* happens in real mode here based on the fnsave and fnstenv images. */
2128	pFpuCtx->CS = 0;
2129	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2130	}
2131	else if (!IEM_IS_LONG_MODE(pVCpu))
2132	{
2133	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2134	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2135	}
2136	else
2137	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2138	}
2139
2140
2141
2142
2143
2144	/**
2145	* Marks the specified stack register as free (for FFREE).
2146	*
2147	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2148	* @param iStReg The register to free.
2149	*/
2150	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
2151	{
2152	Assert(iStReg < 8);
2153	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2154	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2155	pFpuCtx->FTW &= ~RT_BIT(iReg);
2156	}
2157
2158
2159	/**
2160	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2161	*
2162	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2163	*/
2164	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu)
2165	{
2166	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2167	uint16_t uFsw = pFpuCtx->FSW;
2168	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2169	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2170	uFsw &= ~X86_FSW_TOP_MASK;
2171	uFsw \|= uTop;
2172	pFpuCtx->FSW = uFsw;
2173	}
2174
2175
2176	/**
2177	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2178	*
2179	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2180	*/
2181	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu)
2182	{
2183	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2184	uint16_t uFsw = pFpuCtx->FSW;
2185	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2186	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2187	uFsw &= ~X86_FSW_TOP_MASK;
2188	uFsw \|= uTop;
2189	pFpuCtx->FSW = uFsw;
2190	}
2191
2192
2193
2194
2195	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
2196	{
2197	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2198	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2199	if (pFpuCtx->FTW & RT_BIT(iReg))
2200	return VINF_SUCCESS;
2201	return VERR_NOT_FOUND;
2202	}
2203
2204
2205	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
2206	{
2207	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2208	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2209	if (pFpuCtx->FTW & RT_BIT(iReg))
2210	{
2211	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2212	return VINF_SUCCESS;
2213	}
2214	return VERR_NOT_FOUND;
2215	}
2216
2217
2218	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2219	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
2220	{
2221	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2222	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2223	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2224	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2225	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2226	{
2227	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2228	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2229	return VINF_SUCCESS;
2230	}
2231	return VERR_NOT_FOUND;
2232	}
2233
2234
2235	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
2236	{
2237	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2238	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2239	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2240	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2241	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2242	{
2243	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2244	return VINF_SUCCESS;
2245	}
2246	return VERR_NOT_FOUND;
2247	}
2248
2249
2250	/**
2251	* Rotates the stack registers when setting new TOS.
2252	*
2253	* @param pFpuCtx The FPU context.
2254	* @param iNewTop New TOS value.
2255	* @remarks We only do this to speed up fxsave/fxrstor which
2256	* arrange the FP registers in stack order.
2257	* MUST be done before writing the new TOS (FSW).
2258	*/
2259	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop)
2260	{
2261	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2262	RTFLOAT80U ar80Temp[8];
2263
2264	if (iOldTop == iNewTop)
2265	return;
2266
2267	/* Unscrew the stack and get it into 'native' order. */
2268	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2269	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2270	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2271	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2272	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2273	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2274	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2275	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2276
2277	/* Now rotate the stack to the new position. */
2278	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2279	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2280	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2281	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2282	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2283	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2284	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2285	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2286	}
2287
2288
2289	/**
2290	* Updates the FPU exception status after FCW is changed.
2291	*
2292	* @param pFpuCtx The FPU context.
2293	*/
2294	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
2295	{
2296	uint16_t u16Fsw = pFpuCtx->FSW;
2297	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2298	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2299	else
2300	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2301	pFpuCtx->FSW = u16Fsw;
2302	}
2303
2304
2305	/**
2306	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2307	*
2308	* @returns The full FTW.
2309	* @param pFpuCtx The FPU context.
2310	*/
2311	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
2312	{
2313	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2314	uint16_t u16Ftw = 0;
2315	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2316	for (unsigned iSt = 0; iSt < 8; iSt++)
2317	{
2318	unsigned const iReg = (iSt + iTop) & 7;
2319	if (!(u8Ftw & RT_BIT(iReg)))
2320	u16Ftw \|= 3 << (iReg * 2); /* empty */
2321	else
2322	{
2323	uint16_t uTag;
2324	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2325	if (pr80Reg->s.uExponent == 0x7fff)
2326	uTag = 2; /* Exponent is all 1's => Special. */
2327	else if (pr80Reg->s.uExponent == 0x0000)
2328	{
2329	if (pr80Reg->s.uMantissa == 0x0000)
2330	uTag = 1; /* All bits are zero => Zero. */
2331	else
2332	uTag = 2; /* Must be special. */
2333	}
2334	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2335	uTag = 0; /* Valid. */
2336	else
2337	uTag = 2; /* Must be special. */
2338
2339	u16Ftw \|= uTag << (iReg * 2);
2340	}
2341	}
2342
2343	return u16Ftw;
2344	}
2345
2346
2347	/**
2348	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2349	*
2350	* @returns The compressed FTW.
2351	* @param u16FullFtw The full FTW to convert.
2352	*/
2353	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw)
2354	{
2355	uint8_t u8Ftw = 0;
2356	for (unsigned i = 0; i < 8; i++)
2357	{
2358	if ((u16FullFtw & 3) != 3 /empty/)
2359	u8Ftw \|= RT_BIT(i);
2360	u16FullFtw >>= 2;
2361	}
2362
2363	return u8Ftw;
2364	}
2365
2366	/** @} */
2367
2368
2369	/** @name Memory access.
2370	*
2371	* @{
2372	*/
2373
2374
2375	/**
2376	* Checks whether alignment checks are enabled or not.
2377	*
2378	* @returns true if enabled, false if not.
2379	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2380	*/
2381	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu)
2382	{
2383	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2384	return pVCpu->iem.s.uCpl == 3
2385	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2386	}
2387
2388	/**
2389	* Checks if the given segment can be written to, raise the appropriate
2390	* exception if not.
2391	*
2392	* @returns VBox strict status code.
2393	*
2394	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2395	* @param pHid Pointer to the hidden register.
2396	* @param iSegReg The register number.
2397	* @param pu64BaseAddr Where to return the base address to use for the
2398	* segment. (In 64-bit code it may differ from the
2399	* base in the hidden segment.)
2400	*/
2401	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2402	{
2403	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2404
2405	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2406	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2407	else
2408	{
2409	if (!pHid->Attr.n.u1Present)
2410	{
2411	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2412	AssertRelease(uSel == 0);
2413	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2414	return iemRaiseGeneralProtectionFault0(pVCpu);
2415	}
2416
2417	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2418	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2419	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2420	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2421	*pu64BaseAddr = pHid->u64Base;
2422	}
2423	return VINF_SUCCESS;
2424	}
2425
2426
2427	/**
2428	* Checks if the given segment can be read from, raise the appropriate
2429	* exception if not.
2430	*
2431	* @returns VBox strict status code.
2432	*
2433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2434	* @param pHid Pointer to the hidden register.
2435	* @param iSegReg The register number.
2436	* @param pu64BaseAddr Where to return the base address to use for the
2437	* segment. (In 64-bit code it may differ from the
2438	* base in the hidden segment.)
2439	*/
2440	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2441	{
2442	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2443
2444	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2445	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2446	else
2447	{
2448	if (!pHid->Attr.n.u1Present)
2449	{
2450	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2451	AssertRelease(uSel == 0);
2452	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2453	return iemRaiseGeneralProtectionFault0(pVCpu);
2454	}
2455
2456	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2457	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2458	*pu64BaseAddr = pHid->u64Base;
2459	}
2460	return VINF_SUCCESS;
2461	}
2462
2463
2464	/**
2465	* Maps a physical page.
2466	*
2467	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2468	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2469	* @param GCPhysMem The physical address.
2470	* @param fAccess The intended access.
2471	* @param ppvMem Where to return the mapping address.
2472	* @param pLock The PGM lock.
2473	*/
2474	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
2475	{
2476	#ifdef IEM_LOG_MEMORY_WRITES
2477	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2478	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2479	#endif
2480
2481	/** @todo This API may require some improving later. A private deal with PGM
2482	* regarding locking and unlocking needs to be struct. A couple of TLBs
2483	* living in PGM, but with publicly accessible inlined access methods
2484	* could perhaps be an even better solution. */
2485	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2486	GCPhysMem,
2487	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2488	pVCpu->iem.s.fBypassHandlers,
2489	ppvMem,
2490	pLock);
2491	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2492	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2493
2494	return rc;
2495	}
2496
2497
2498	/**
2499	* Unmap a page previously mapped by iemMemPageMap.
2500	*
2501	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2502	* @param GCPhysMem The physical address.
2503	* @param fAccess The intended access.
2504	* @param pvMem What iemMemPageMap returned.
2505	* @param pLock The PGM lock.
2506	*/
2507	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
2508	{
2509	NOREF(pVCpu);
2510	NOREF(GCPhysMem);
2511	NOREF(fAccess);
2512	NOREF(pvMem);
2513	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2514	}
2515
2516	#ifdef IEM_WITH_SETJMP
2517
2518	/** @todo slim this down */
2519	DECLINLINE(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2520	{
2521	Assert(cbMem >= 1);
2522	Assert(iSegReg < X86_SREG_COUNT);
2523
2524	/*
2525	* 64-bit mode is simpler.
2526	*/
2527	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2528	{
2529	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2530	{
2531	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2532	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2533	GCPtrMem += pSel->u64Base;
2534	}
2535
2536	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2537	return GCPtrMem;
2538	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2539	}
2540	/*
2541	* 16-bit and 32-bit segmentation.
2542	*/
2543	else if (iSegReg != UINT8_MAX)
2544	{
2545	/** @todo Does this apply to segments with 4G-1 limit? */
2546	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2547	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2548	{
2549	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2550	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2551	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2552	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2553	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2554	\| X86_SEL_TYPE_CODE))
2555	{
2556	case X86DESCATTR_P: /* readonly data, expand up */
2557	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2558	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2559	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2560	/* expand up */
2561	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2562	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2563	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2564	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2565	break;
2566
2567	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2568	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2569	/* expand down */
2570	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2571	&& ( pSel->Attr.n.u1DefBig
2572	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2573	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2574	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2575	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2576	break;
2577
2578	default:
2579	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2580	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2581	break;
2582	}
2583	}
2584	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2585	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2586	}
2587	/*
2588	* 32-bit flat address.
2589	*/
2590	else
2591	return GCPtrMem;
2592	}
2593
2594
2595	/** @todo slim this down */
2596	DECLINLINE(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2597	{
2598	Assert(cbMem >= 1);
2599	Assert(iSegReg < X86_SREG_COUNT);
2600
2601	/*
2602	* 64-bit mode is simpler.
2603	*/
2604	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2605	{
2606	if (iSegReg >= X86_SREG_FS)
2607	{
2608	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2609	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2610	GCPtrMem += pSel->u64Base;
2611	}
2612
2613	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2614	return GCPtrMem;
2615	}
2616	/*
2617	* 16-bit and 32-bit segmentation.
2618	*/
2619	else
2620	{
2621	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2622	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2623	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2624	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2625	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2626	{
2627	/* expand up */
2628	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2629	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2630	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2631	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2632	}
2633	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2634	{
2635	/* expand down */
2636	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2637	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2638	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2639	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2640	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2641	}
2642	else
2643	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2644	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2645	}
2646	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2647	}
2648
2649	#endif /* IEM_WITH_SETJMP */
2650
2651	/**
2652	* Fakes a long mode stack selector for SS = 0.
2653	*
2654	* @param pDescSs Where to return the fake stack descriptor.
2655	* @param uDpl The DPL we want.
2656	*/
2657	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
2658	{
2659	pDescSs->Long.au64[0] = 0;
2660	pDescSs->Long.au64[1] = 0;
2661	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2662	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2663	pDescSs->Long.Gen.u2Dpl = uDpl;
2664	pDescSs->Long.Gen.u1Present = 1;
2665	pDescSs->Long.Gen.u1Long = 1;
2666	}
2667
2668	/** @} */
2669
2670
2671	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2672
2673	/**
2674	* Gets CR0 fixed-0 bits in VMX non-root mode.
2675	*
2676	* We do this rather than fetching what we report to the guest (in
2677	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2678	* values regardless of whether unrestricted-guest feature is available on the CPU.
2679	*
2680	* @returns CR0 fixed-0 bits.
2681	* @param pVCpu The cross context virtual CPU structure.
2682	*/
2683	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu)
2684	{
2685	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2686	Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
2687
2688	static uint64_t const s_auCr0Fixed0[2] = { VMX_V_CR0_FIXED0, VMX_V_CR0_FIXED0_UX };
2689	PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
2690	uint8_t const fUnrestrictedGuest = !!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
2691	uint64_t const uCr0Fixed0 = s_auCr0Fixed0[fUnrestrictedGuest];
2692	Assert(!(uCr0Fixed0 & (X86_CR0_NW \| X86_CR0_CD)));
2693	return uCr0Fixed0;
2694	}
2695
2696
2697	/**
2698	* Sets virtual-APIC write emulation as pending.
2699	*
2700	* @param pVCpu The cross context virtual CPU structure.
2701	* @param offApic The offset in the virtual-APIC page that was written.
2702	*/
2703	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic)
2704	{
2705	Assert(offApic < XAPIC_OFF_END + 4);
2706
2707	/*
2708	* Record the currently updated APIC offset, as we need this later for figuring
2709	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2710	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2711	*/
2712	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2713
2714	/*
2715	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2716	* virtualization or APIC-write emulation).
2717	*/
2718	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2719	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2720	}
2721
2722	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2723
2724	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 97406

Download in other formats: