IEMInline-armv8.h

Last change on this file was 108791, checked in by vboxsync, 11 days ago
VMM/IEM: More ARM target work. jiraref:VBP-1598
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 102.4 KB

Line
1	/* $Id: IEMInline-armv8.h 108791 2025-03-28 21:58:31Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions, ARMv8 target.
4	*/
5
6	/*
7	* Copyright (C) 2011-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_VMMAll_target_armv8_IEMInline_armv8_h
29	#define VMM_INCLUDED_SRC_VMMAll_target_armv8_IEMInline_armv8_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include <VBox/err.h>
35	#include <iprt/armv8.h>
36
37
38	/**
39	* Calculates the IEM_F_ARM_A & IEM_F_ARM_AA flags.
40	*
41	* @returns Mix of IEM_F_ARM_A, IEM_F_ARM_AA and zero.
42	* @param pVCpu The cross context virtual CPU structure of the
43	* calling thread.
44	* @param fExecMode The mode part of fExec.
45	*/
46	DECL_FORCE_INLINE(uint32_t) iemCalcExecAcFlag(PVMCPUCC pVCpu, uint32_t fExecMode) RT_NOEXCEPT
47	{
48	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_SYSREG_EL2 \| CPUMCTX_EXTRN_PSTATE);
49	uint64_t const fSctlr = IEM_F_MODE_ARM_GET_EL(fExecMode) == 1
50	\|\| ( IEM_F_MODE_ARM_GET_EL(fExecMode) == 0
51	&& (pVCpu->cpum.GstCtx.HcrEl2.u64 & (RT_BIT_64(34)/E2H/ \| RT_BIT_64(27)/TGE/))
52	!= (RT_BIT_64(34)/E2H/ \| RT_BIT_64(27)/TGE/) )
53	? pVCpu->cpum.GstCtx.Sctlr.u64
54	: pVCpu->cpum.GstCtx.SctlrEl2.u64;
55	/** @todo armv8: EL3 */
56	AssertCompile(ARMV8_SCTLR_EL1_A == ARMV8_SCTLR_EL2_A);
57	AssertCompile(ARMV8_SCTLR_EL1_NAA == ARMV8_SCTLR_EL2_NAA);
58
59	return ((fSctlr & ARMV8_SCTLR_EL1_A) ? IEM_F_ARM_A : 0)
60	\| ((fSctlr & ARMV8_SCTLR_EL1_NAA) ? 0 : IEM_F_ARM_AA);
61	}
62
63
64	/**
65	* Calculates the stage 1 page size.
66	*
67	* @returns IEM_F_ARM_S1_PAGE_MASK
68	* @param pVCpu The cross context virtual CPU structure of the
69	* calling thread.
70	* @param fExecMode The mode part of fExec.
71	*/
72	DECL_FORCE_INLINE(uint32_t) iemCalcExecStage1PageSize(PVMCPUCC pVCpu, uint32_t fExecMode) RT_NOEXCEPT
73	{
74	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SCTLR_TCR_TTBR \| CPUMCTX_EXTRN_PSTATE);
75	uint64_t const fSCtlR = IEM_F_MODE_ARM_GET_EL(fExecMode) <= 1 ? pVCpu->cpum.GstCtx.Sctlr.u64
76	#if 1 /** @todo armv8: EL3 & check up EL2 logic. */
77	: pVCpu->cpum.GstCtx.SctlrEl2.u64;
78	#else
79	: IEM_F_MODE_ARM_GET_EL(fExecMode) == 2 ? pVCpu->cpum.GstCtx.SctlrEl2.u64
80	: pVCpu->cpum.GstCtx.SctlrEl3.u64;
81	#endif
82	if (fSCtlR & ARMV8_SCTLR_EL1_M)
83	{
84	uint64_t const fTcr = IEM_F_MODE_ARM_GET_EL(fExecMode) <= 1 ? pVCpu->cpum.GstCtx.Tcr.u64
85	#if 1 /** @todo armv8: EL3 & check up EL2 logic. */
86	: pVCpu->cpum.GstCtx.TcrEl2.u64;
87	#else
88	: IEM_F_MODE_ARM_GET_EL(fExecMode) == 2 ? pVCpu->cpum.GstCtx.TcrEl2.u64
89	: pVCpu->cpum.GstCtx.TcrEl3.u64;
90	#endif
91	switch (fTcr & ARMV8_TCR_EL1_AARCH64_TG0_MASK)
92	{
93	case ARMV8_TCR_EL1_AARCH64_TG0_4KB << ARMV8_TCR_EL1_AARCH64_TG0_SHIFT:
94	return IEM_F_ARM_S1_PAGE_4K;
95	case ARMV8_TCR_EL1_AARCH64_TG0_16KB << ARMV8_TCR_EL1_AARCH64_TG0_SHIFT:
96	return IEM_F_ARM_S1_PAGE_16K;
97	case ARMV8_TCR_EL1_AARCH64_TG0_64KB << ARMV8_TCR_EL1_AARCH64_TG0_SHIFT:
98	return IEM_F_ARM_S1_PAGE_64K;
99	default:
100	AssertFailed();
101	return IEM_F_ARM_S1_PAGE_4K;
102	}
103	}
104	/* MMU is not enabled, use 4KB TLB entries for now. */
105	/** @todo check out 64KB for non-MMU mode. */
106	return IEM_F_ARM_S1_PAGE_4K; /** @todo Do we need a NO_MMU flag? */
107	}
108
109
110	/**
111	* Calculates the IEM_F_MODE_XXX, IEM_F_ARM_A, IEM_F_ARM_AA and IEM_F_ARM_SP_IDX
112	*
113	* @returns IEM_F_MODE_XXX, IEM_F_ARM_A, IEM_F_ARM_AA, IEM_F_ARM_SP_IDX.
114	* @param pVCpu The cross context virtual CPU structure of the
115	* calling thread.
116	*/
117	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndSpIdxAndAcFlagsAndS1PgSize(PVMCPUCC pVCpu) RT_NOEXCEPT
118	{
119	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_PSTATE);
120
121	/** @todo we probably need to take the security state into the mode as
122	* well... */
123
124	/* EL, SP idx, aarch64, aarch32, t32. */
125	AssertCompile(ARMV8_SPSR_EL2_AARCH64_EL == IEM_F_MODE_ARM_EL_MASK);
126	AssertCompile(ARMV8_SPSR_EL2_AARCH64_M4 == IEM_F_MODE_ARM_32BIT);
127	AssertCompile(ARMV8_SPSR_EL2_AARCH64_T == IEM_F_MODE_ARM_T32);
128	uint32_t fExec = (uint32_t)pVCpu->cpum.GstCtx.fPState
129	& (uint32_t)(ARMV8_SPSR_EL2_AARCH64_EL \| ARMV8_SPSR_EL2_AARCH64_M4 \| ARMV8_SPSR_EL2_AARCH64_T);
130	if (!(fExec & ARMV8_SPSR_EL2_AARCH64_M4))
131	{
132	Assert(!(fExec & ARMV8_SPSR_EL2_AARCH64_T)); /* aarch64 */
133	if (pVCpu->cpum.GstCtx.fPState & ARMV8_SPSR_EL2_AARCH64_SP)
134	fExec \|= IEM_F_MODE_ARM_GET_EL(fExec);
135	}
136	else
137	{
138	fExec &= ARMV8_SPSR_EL2_AARCH64_M4 \| ARMV8_SPSR_EL2_AARCH64_T;
139	switch (pVCpu->cpum.GstCtx.fPState & ARMV8_SPSR_EL2_AARCH64_M)
140	{
141	case 0x0: /* User */
142	//fExec \|= 0 << IEM_F_MODE_ARM_EL_SHIFT;
143	break;
144	case 0x1: /* FIQ */
145	case 0x2: /* IRQ */
146	case 0x3: /* Supervisor */
147	case 0x7: /* Abort */
148	case 0xb: /* Undefined */
149	case 0xf: /* System */
150	fExec \|= 1 << IEM_F_MODE_ARM_EL_SHIFT;
151	break;
152	case 0xa: /* Hypervisor */
153	fExec \|= 2 << IEM_F_MODE_ARM_EL_SHIFT;
154	break;
155	case 0x4: case 0x5: case 0x6:
156	case 0x8: case 0x9:
157	case 0xc: case 0xd: case 0xe:
158	AssertFailed();
159	break;
160	}
161
162	#if 0 /** @todo SP index for aarch32: We don't have SPSEL. */
163	if (pVCpu->cpum.GstCtx.SpSel.u32 & 1)
164	fExec \|= IEM_F_MODE_ARM_GET_EL(fExec);
165	#endif
166	}
167
168	/* Alignment checks: */
169	fExec \|= iemCalcExecAcFlag(pVCpu, fExec);
170
171	/* Page size. */
172	fExec \|= iemCalcExecStage1PageSize(pVCpu, fExec);
173	return fExec;
174	}
175
176	#ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */
177
178	/**
179	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) and IEM_F_ARM_SOFTWARE_STEP flags.
180	*
181	* @returns IEM_F_BRK_PENDING_XXX + IEM_F_ARM_SOFTWARE_STEP.
182	* @param pVCpu The cross context virtual CPU structure of the
183	* calling thread.
184	*/
185	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
186	{
187	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_PSTATE);
188	/** @todo The debug state is a bit complicated at first. It appears the
189	* MDSCR_EL1.SS flag is the one to set/disable together with anything
190	* masking debug exceptions. The PSTATE.SS flag is just to indicate
191	* whether we're raising a debug exception before the current
192	* instruction (SS=1) or the next one (SS=0, set to 1 upon instruction
193	* retirement). More on the exact boundrary and priority stuff needs
194	* to be considered here... */
195	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.Mdscr.u64 & ARMV8_MDSCR_EL1_AARCH64_SS)
196	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
197	return 0;
198	return iemCalcExecDbgFlagsSlow(pVCpu);
199	}
200
201
202	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
203	{
204	return iemCalcExecModeAndSpIdxAndAcFlagsAndS1PgSize(pVCpu)
205	\| iemCalcExecDbgFlags(pVCpu)
206	;
207	}
208
209
210	/**
211	* Re-calculates the IEM_F_MODE_XXX, IEM_F_ARM_A, IEM_F_ARM_AA and
212	* IEM_F_ARM_SP_IDX parts of IEMCPU::fExec.
213	*
214	* @param pVCpu The cross context virtual CPU structure of the
215	* calling thread.
216	*/
217	DECL_FORCE_INLINE(void) iemRecalcExecModeAndSpIdxAndAcFlags(PVMCPUCC pVCpu)
218	{
219	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_ARM_A \| IEM_F_ARM_AA))
220	\| iemCalcExecModeAndSpIdxAndAcFlagsAndS1PgSize(pVCpu);
221	}
222
223
224	/**
225	* Re-calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) and
226	* IEM_F_ARM_SOFTWARE_STEP part of IEMCPU::fExec.
227	*
228	* @param pVCpu The cross context virtual CPU structure of the
229	* calling thread.
230	*/
231	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
232	{
233	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_PENDING_BRK_MASK \| IEM_F_ARM_SOFTWARE_STEP))
234	\| iemCalcExecDbgFlags(pVCpu);
235	}
236
237	#endif /* VBOX_INCLUDED_vmm_dbgf_h */
238
239
240	/** @name Register Access.
241	* @{
242	*/
243
244
245	/**
246	* Fetches the value of a 8-bit general purpose register.
247	*
248	* @returns The register value.
249	* @param pVCpu The cross context virtual CPU structure of the calling thread.
250	* @param iReg The register.
251	* @param fSp Set if register 31 is SP, otherwise it's zero.
252	*/
253	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg, bool fSp) RT_NOEXCEPT
254	{
255	Assert(iReg < 32);
256	return iReg < 31 ? (uint8_t)pVCpu->cpum.GstCtx.aGRegs[iReg].w
257	: fSp ? (uint8_t)pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u32
258	: UINT8_C(0);
259	}
260
261
262	/**
263	* Fetches the value of a 16-bit general purpose register.
264	*
265	* @returns The register value.
266	* @param pVCpu The cross context virtual CPU structure of the calling thread.
267	* @param iReg The register.
268	* @param fSp Set if register 31 is SP, otherwise it's zero.
269	*/
270	DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg, bool fSp) RT_NOEXCEPT
271	{
272	Assert(iReg < 32);
273	return iReg < 31 ? (uint16_t)pVCpu->cpum.GstCtx.aGRegs[iReg].w
274	: fSp ? (uint16_t)pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u32
275	: UINT16_C(0);
276	}
277
278
279	/**
280	* Fetches the value of a 32-bit general purpose register.
281	*
282	* @returns The register value.
283	* @param pVCpu The cross context virtual CPU structure of the calling thread.
284	* @param iReg The register.
285	* @param fSp Set if register 31 is SP, otherwise it's zero.
286	*/
287	DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg, bool fSp) RT_NOEXCEPT
288	{
289	Assert(iReg < 32);
290	return iReg < 31 ? pVCpu->cpum.GstCtx.aGRegs[iReg].w
291	: fSp ? pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u32
292	: UINT32_C(0);
293	}
294
295
296	/**
297	* Fetches the value of a 64-bit general purpose register.
298	*
299	* @returns The register value.
300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
301	* @param iReg The register.
302	* @param fSp Set if register 31 is SP, otherwise it's zero.
303	*/
304	DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg, bool fSp) RT_NOEXCEPT
305	{
306	Assert(iReg < 32);
307	return iReg < 31 ? pVCpu->cpum.GstCtx.aGRegs[iReg].x
308	: fSp ? pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u64
309	: UINT64_C(0);
310	}
311
312
313	/**
314	* Stores a 8-bit value to a general purpose register, zeroing extending it to
315	* the full register width.
316	*
317	* @param pVCpu The cross context virtual CPU structure of the calling thread.
318	* @param iReg The register.
319	* @param fSp Set if register 31 is SP, otherwise it's zero.
320	* @param uValue The value to store.
321	*/
322	DECL_FORCE_INLINE(void) iemGRegStoreU8(PVMCPUCC pVCpu, uint8_t iReg, bool fSp, uint8_t uValue) RT_NOEXCEPT
323	{
324	Assert(iReg < 32);
325	if (iReg < 31)
326	pVCpu->cpum.GstCtx.aGRegs[iReg].x = uValue;
327	else if (fSp)
328	pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u64 = uValue;
329	}
330
331
332	/**
333	* Stores a 16-bit value to a general purpose register, zeroing extending it to
334	* the full register width.
335	*
336	* @param pVCpu The cross context virtual CPU structure of the calling thread.
337	* @param iReg The register.
338	* @param fSp Set if register 31 is SP, otherwise it's zero.
339	* @param uValue The value to store.
340	*/
341	DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, bool fSp, uint16_t uValue) RT_NOEXCEPT
342	{
343	Assert(iReg < 32);
344	if (iReg < 31)
345	pVCpu->cpum.GstCtx.aGRegs[iReg].x = uValue;
346	else if (fSp)
347	pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u64 = uValue;
348	}
349
350
351	/**
352	* Stores a 32-bit value to a general purpose register, zeroing extending it to
353	* the full register width.
354	*
355	* @param pVCpu The cross context virtual CPU structure of the calling thread.
356	* @param iReg The register.
357	* @param fSp Set if register 31 is SP, otherwise it's zero.
358	* @param uValue The value to store.
359	*/
360	DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, bool fSp, uint32_t uValue) RT_NOEXCEPT
361	{
362	Assert(iReg < 32);
363	if (iReg < 31)
364	pVCpu->cpum.GstCtx.aGRegs[iReg].x = uValue;
365	else if (fSp)
366	pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u64 = uValue;
367	}
368
369
370	/**
371	* Stores a 64-bit value to a general purpose register.
372	*
373	* @param pVCpu The cross context virtual CPU structure of the calling thread.
374	* @param iReg The register.
375	* @param fSp Set if register 31 is SP, otherwise it's zero.
376	* @param uValue The value to store.
377	*/
378	DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, bool fSp, uint64_t uValue) RT_NOEXCEPT
379	{
380	Assert(iReg < 32);
381	if (iReg < 31)
382	pVCpu->cpum.GstCtx.aGRegs[iReg].x = uValue;
383	else if (fSp)
384	pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u64 = uValue;
385	}
386
387
388	/**
389	* Get the address of the top of the stack.
390	*
391	* @param pVCpu The cross context virtual CPU structure of the calling thread.
392	*/
393	DECL_FORCE_INLINE(RTGCPTR) iemRegGetSp(PCVMCPU pVCpu) RT_NOEXCEPT
394	{
395	return pVCpu->cpum.GstCtx.aSpReg[IEM_F_ARM_GET_SP_IDX(pVCpu->iem.s.fExec)].u64;
396	}
397
398
399	/**
400	* Updates the PC to point to the next instruction.
401	*
402	* This is the generic version used by code that isn't mode specific. Code that
403	* is only used in aarch64, aarch32 and t32 should call the specific versions
404	* below.
405	*
406	* @param pVCpu The cross context virtual CPU structure of the calling thread.
407	* @param f32Bit Set if it's a 32-bit wide instruction, clear if 16-bit (T32
408	* mode only).
409	* @see iemRegPcIncA64, iemRegPcIncA32, iemRegPcIncT32
410	*/
411	DECL_FORCE_INLINE(void) iemRegPcInc(PVMCPUCC pVCpu, bool f32Bit = true) RT_NOEXCEPT
412	{
413	if (!(pVCpu->iem.s.fExec & IEM_F_MODE_ARM_32BIT))
414	{
415	Assert(f32Bit);
416	pVCpu->cpum.GstCtx.Pc.u64 = pVCpu->cpum.GstCtx.Pc.u64 + 4;
417	}
418	else
419	{
420	Assert(f32Bit \|\| (pVCpu->iem.s.fExec & IEM_F_MODE_ARM_T32));
421	pVCpu->cpum.GstCtx.Pc.u64 = pVCpu->cpum.GstCtx.Pc.u32 + (f32Bit ? 4 : 2);
422	}
423	}
424
425
426	/**
427	* Updates the PC to point to the next instruction in aarch64 mode.
428	*
429	* @param pVCpu The cross context virtual CPU structure of the calling thread.
430	*/
431	DECL_FORCE_INLINE(void) iemRegPcIncA64(PVMCPUCC pVCpu) RT_NOEXCEPT
432	{
433	pVCpu->cpum.GstCtx.Pc.u64 += 4;
434	}
435
436
437	/**
438	* Updates the PC to point to the next instruction in aarch32 mode.
439	*
440	* @param pVCpu The cross context virtual CPU structure of the calling thread.
441	*/
442	DECL_FORCE_INLINE(void) iemRegPcIncA32(PVMCPUCC pVCpu) RT_NOEXCEPT
443	{
444	pVCpu->cpum.GstCtx.Pc.u64 = pVCpu->cpum.GstCtx.Pc.u32 + 4;
445	}
446
447
448	/**
449	* Updates the PC to point to the next instruction in T32 mode.
450	*
451	* @param pVCpu The cross context virtual CPU structure of the calling thread.
452	* @param f32Bit Set if it's a 32-bit wide instruction.
453	*/
454	DECL_FORCE_INLINE(void) iemRegPcIncT32(PVMCPUCC pVCpu, bool f32Bit = false) RT_NOEXCEPT
455	{
456	pVCpu->cpum.GstCtx.Pc.u64 = pVCpu->cpum.GstCtx.Pc.u32 + (!f32Bit ? 2 : 4);
457	}
458
459
460	/**
461	* Gets the exeception level that debug exceptions are routed to.
462	*
463	* @returns Exception level.
464	* @param pVCpu The cross context virtual CPU structure of the calling thread.
465	*/
466	DECL_FORCE_INLINE(uint8_t) iemGetDebugExceptionLevel(PVMCPUCC pVCpu)
467	{
468	/** @todo EL3 */
469	if ( (pVCpu->cpum.GstCtx.MdcrEl2.u64 & ARMV8_MDCR_EL2_TDE)
470	\|\| (pVCpu->cpum.GstCtx.HcrEl2.u64 & ARMV8_HCR_EL2_TGE))
471	return 2;
472	return 1;
473	}
474
475
476	/**
477	* Called to handle software step when retiring an instruction.
478	*
479	* This is only called when IEM_F_ARM_SOFTWARE_STEP is set.
480	*/
481	static VBOXSTRICTRC iemFinishInstructionWithSoftwareStep(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
482	{
483	/*
484	* The ARMV8_MDSCR_EL1_AARCH64_SS flag must be set. Any instruction
485	* modifying it will recalculate fExec, so we shouldn't get here.
486	*/
487	Assert(pVCpu->cpum.GstCtx.Mdscr.u64 & ARMV8_MDSCR_EL1_AARCH64_SS);
488
489	/*
490	* Clear the PSTATE.SS.
491	*/
492	pVCpu->cpum.GstCtx.fPState &= ~ARMV8_SPSR_EL2_AARCH64_SS;
493	/** @todo guess IEM_F_ARM_SOFTWARE_STEP shouldn't be cleared till
494	* ARMV8_MDSCR_EL1_AARCH64_SS is cleared... Re-check that against
495	* docs and what guests actually does. */
496
497	/*
498	* Check if we can raise the exception.
499	*/
500	/** @todo The documentation (D2.3.1) seems to indicate that PSTATE.D=1 won't
501	* mask software step debug exceptions when bCurEl is less than bDebugEl.
502	* mask SS. But I find that hard to believe... */
503	if (!(pVCpu->cpum.GstCtx.fPState & ARMV8_SPSR_EL2_AARCH64_D))
504	{
505	uint8_t const bDebugEl = iemGetDebugExceptionLevel(pVCpu);
506	uint8_t const bCurEl = IEM_F_MODE_ARM_GET_EL(pVCpu->iem.s.fExec);
507	if ( bCurEl < bDebugEl
508	\|\| ( bCurEl == bDebugEl
509	&& (pVCpu->cpum.GstCtx.Mdscr.u64 & ARMV8_MDSCR_EL1_AARCH64_KDE)) )
510	{
511	LogFlowFunc(("Guest debug exception/software step at %016llx\n", pVCpu->cpum.GstCtx.Pc.u64));
512	/** @todo iemRaiseDebugException(pVCpu, xxx); */
513	AssertFailedReturn(VERR_IEM_ASPECT_NOT_IMPLEMENTED);
514	}
515	}
516	return rcNormal;
517	}
518
519
520	/**
521	* Deals with PSTATE.SS as necessary, maybe raising debug exception.
522	*
523	* @param pVCpu The cross context virtual CPU structure of the calling thread.
524	* @param rcNormal VINF_SUCCESS to continue TB.
525	* VINF_IEM_REEXEC_BREAK to force TB exit when
526	* taking the wrong conditional branhc.
527	*/
528	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
529	{
530	/*
531	* We assume that most of the time nothing actually needs doing here.
532	*/
533	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_ARM_SOFTWARE_STEP)))
534	return rcNormal;
535	return iemFinishInstructionWithSoftwareStep(pVCpu, rcNormal);
536	}
537
538
539	/**
540	* Updates the PC to point to the next instruction and deals with PSTATE.SS.
541	*
542	* This is the generic version used by code that isn't mode specific. Code that
543	* is only used in aarch64, aarch32 and t32 should call the specific versions
544	* below.
545	*
546	* @param pVCpu The cross context virtual CPU structure of the calling thread.
547	* @param rcNormal VINF_SUCCESS to continue TB.
548	* VINF_IEM_REEXEC_BREAK to force TB exit when
549	* taking the wrong conditional branhc.
550	*/
551	DECL_FORCE_INLINE(VBOXSTRICTRC)
552	iemRegPcIncAndFinishClearingFlags(PVMCPUCC pVCpu, bool f32Bit = true, int rcNormal = VINF_SUCCESS) RT_NOEXCEPT
553	{
554	iemRegPcInc(pVCpu, f32Bit);
555	return iemRegFinishClearingFlags(pVCpu, rcNormal);
556	}
557
558
559	/**
560	* Updates the PC to point to the next AArch64 instruction (32-bit) for mode and
561	* deals with PSTATE.SS.
562	*
563	* @param pVCpu The cross context virtual CPU structure of the calling thread.
564	* @param rcNormal VINF_SUCCESS to continue TB.
565	* VINF_IEM_REEXEC_BREAK to force TB exit when
566	* taking the wrong conditional branhc.
567	*/
568	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegPcIncA64AndFinishingClearingFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
569	{
570	iemRegPcIncA64(pVCpu);
571	return iemRegFinishClearingFlags(pVCpu, rcNormal);
572	}
573
574
575	/**
576	* Updates the PC to point to the next AArch32 instruction (32-bit) for mode and
577	* deals with PSTATE.SS.
578	*
579	* @param pVCpu The cross context virtual CPU structure of the calling thread.
580	* @param rcNormal VINF_SUCCESS to continue TB.
581	* VINF_IEM_REEXEC_BREAK to force TB exit when
582	* taking the wrong conditional branhc.
583	*/
584	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegPcIncA32AndFinishingClearingFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
585	{
586	iemRegPcIncA32(pVCpu);
587	return iemRegFinishClearingFlags(pVCpu, rcNormal);
588	}
589
590
591	/**
592	* Updates the PC to point to the next thumb instruction (16-bit or 32-bit) for
593	* mode and deals with PSTATE.SS.
594	*
595	* @param pVCpu The cross context virtual CPU structure of the calling thread.
596	* @param f32Bit Set if it's a 32-bit wide instruction.
597	* @param rcNormal VINF_SUCCESS to continue TB.
598	* VINF_IEM_REEXEC_BREAK to force TB exit when
599	* taking the wrong conditional branhc.
600	*/
601	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegPcIncT32AndFinishingClearingFlags(PVMCPUCC pVCpu, bool f32Bit, int rcNormal) RT_NOEXCEPT
602	{
603	iemRegPcIncT32(pVCpu, f32Bit);
604	return iemRegFinishClearingFlags(pVCpu, rcNormal);
605	}
606
607
608	/**
609	* Tail method for a finish function that does't clear flags nor raises any
610	* debug exceptions.
611	*
612	* @param pVCpu The cross context virtual CPU structure of the calling thread.
613	* @param rcNormal VINF_SUCCESS to continue TB.
614	* VINF_IEM_REEXEC_BREAK to force TB exit when
615	* taking the wrong conditional branhc.
616	*/
617	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishNoFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
618	{
619	Assert(!(pVCpu->cpum.GstCtx.Mdscr.u64 & ARMV8_MDSCR_EL1_AARCH64_SS));
620	Assert(!(pVCpu->iem.s.fExec & IEM_F_ARM_SOFTWARE_STEP));
621
622	RT_NOREF(pVCpu);
623	return rcNormal;
624	}
625
626
627	/**
628	* Updates the PC to point to the next AArch64 instruction (32-bit) for mode,
629	* skipping PSTATE.SS as it's assumed to be zero or otherwise left alone.
630	*
631	* @param pVCpu The cross context virtual CPU structure of the calling thread.
632	* @param rcNormal VINF_SUCCESS to continue TB.
633	* VINF_IEM_REEXEC_BREAK to force TB exit when
634	* taking the wrong conditional branhc.
635	*/
636	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegPcIncA64AndFinishingNoFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
637	{
638	iemRegPcIncA64(pVCpu);
639	return iemRegFinishNoFlags(pVCpu, rcNormal);
640	}
641
642
643	/**
644	* Updates the PC to point to the next AArch32 instruction (32-bit) for mode,
645	* skipping PSTATE.SS as it's assumed to be zero or otherwise left alone.
646	*
647	* @param pVCpu The cross context virtual CPU structure of the calling thread.
648	* @param rcNormal VINF_SUCCESS to continue TB.
649	* VINF_IEM_REEXEC_BREAK to force TB exit when
650	* taking the wrong conditional branhc.
651	*/
652	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegPcIncA32AndFinishingNoFlags(PVMCPUCC pVCpu, int rcNormal) RT_NOEXCEPT
653	{
654	iemRegPcIncA32(pVCpu);
655	return iemRegFinishNoFlags(pVCpu, rcNormal);
656	}
657
658
659	/**
660	* Updates the PC to point to the next thumb instruction (16-bit or 32-bit) for
661	* mode, skipping PSTATE.SS as it's assumed to be zero or otherwise left alone.
662	*
663	* @param pVCpu The cross context virtual CPU structure of the calling thread.
664	* @param f32Bit Set if it's a 32-bit wide instruction.
665	* @param rcNormal VINF_SUCCESS to continue TB.
666	* VINF_IEM_REEXEC_BREAK to force TB exit when
667	* taking the wrong conditional branhc.
668	*/
669	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegPcIncT32AndFinishingNoFlags(PVMCPUCC pVCpu, bool f32Bit, int rcNormal) RT_NOEXCEPT
670	{
671	iemRegPcIncT32(pVCpu, f32Bit);
672	return iemRegFinishNoFlags(pVCpu, rcNormal);
673	}
674
675
676	#if 0 /** @todo go over this later */
677
678
679	/**
680	* Adds a 8-bit signed jump offset to RIP from 64-bit code.
681	*
682	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
683	* segment limit.
684	*
685	* @param pVCpu The cross context virtual CPU structure of the calling thread.
686	* @param cbInstr Instruction size.
687	* @param offNextInstr The offset of the next instruction.
688	* @param enmEffOpSize Effective operand size.
689	* @param rcNormal VINF_SUCCESS to continue TB.
690	* VINF_IEM_REEXEC_BREAK to force TB exit when
691	* taking the wrong conditional branhc.
692	*/
693	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
694	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
695	{
696	Assert(IEM_IS_64BIT_CODE(pVCpu));
697	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
698
699	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
700	if (enmEffOpSize == IEMMODE_16BIT)
701	uNewRip &= UINT16_MAX;
702
703	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
704	pVCpu->cpum.GstCtx.rip = uNewRip;
705	else
706	return iemRaiseGeneralProtectionFault0(pVCpu);
707
708	#ifndef IEM_WITH_CODE_TLB
709	iemOpcodeFlushLight(pVCpu, cbInstr);
710	#endif
711
712	/*
713	* Clear RF and finish the instruction (maybe raise #DB).
714	*/
715	return iemRegFinishClearingFlags(pVCpu, rcNormal);
716	}
717
718
719	/**
720	* Adds a 8-bit signed jump offset to RIP from 64-bit code when the caller is
721	* sure it stays within the same page.
722	*
723	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
724	* segment limit.
725	*
726	* @param pVCpu The cross context virtual CPU structure of the calling thread.
727	* @param cbInstr Instruction size.
728	* @param offNextInstr The offset of the next instruction.
729	* @param enmEffOpSize Effective operand size.
730	* @param rcNormal VINF_SUCCESS to continue TB.
731	* VINF_IEM_REEXEC_BREAK to force TB exit when
732	* taking the wrong conditional branhc.
733	*/
734	DECL_FORCE_INLINE(VBOXSTRICTRC)
735	iemRegRip64RelativeJumpS8IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
736	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
737	{
738	Assert(IEM_IS_64BIT_CODE(pVCpu));
739	Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize);
740
741	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
742	Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));
743	pVCpu->cpum.GstCtx.rip = uNewRip;
744
745	#ifndef IEM_WITH_CODE_TLB
746	iemOpcodeFlushLight(pVCpu, cbInstr);
747	#endif
748
749	/*
750	* Clear RF and finish the instruction (maybe raise #DB).
751	*/
752	return iemRegFinishClearingFlags(pVCpu, rcNormal);
753	}
754
755
756	/**
757	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
758	* code (never 64-bit).
759	*
760	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
761	* segment limit.
762	*
763	* @param pVCpu The cross context virtual CPU structure of the calling thread.
764	* @param cbInstr Instruction size.
765	* @param offNextInstr The offset of the next instruction.
766	* @param enmEffOpSize Effective operand size.
767	* @param rcNormal VINF_SUCCESS to continue TB.
768	* VINF_IEM_REEXEC_BREAK to force TB exit when
769	* taking the wrong conditional branhc.
770	*/
771	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
772	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
773	{
774	Assert(!IEM_IS_64BIT_CODE(pVCpu));
775	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
776
777	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
778	if (enmEffOpSize == IEMMODE_16BIT)
779	uNewEip &= UINT16_MAX;
780	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
781	pVCpu->cpum.GstCtx.rip = uNewEip;
782	else
783	return iemRaiseGeneralProtectionFault0(pVCpu);
784
785	#ifndef IEM_WITH_CODE_TLB
786	iemOpcodeFlushLight(pVCpu, cbInstr);
787	#endif
788
789	/*
790	* Clear RF and finish the instruction (maybe raise #DB).
791	*/
792	return iemRegFinishClearingFlags(pVCpu, rcNormal);
793	}
794
795
796	/**
797	* Adds a 8-bit signed jump offset to EIP, on 386 or later from FLAT 32-bit code
798	* (never 64-bit).
799	*
800	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
801	* segment limit.
802	*
803	* @param pVCpu The cross context virtual CPU structure of the calling thread.
804	* @param cbInstr Instruction size.
805	* @param offNextInstr The offset of the next instruction.
806	* @param enmEffOpSize Effective operand size.
807	* @param rcNormal VINF_SUCCESS to continue TB.
808	* VINF_IEM_REEXEC_BREAK to force TB exit when
809	* taking the wrong conditional branhc.
810	*/
811	DECL_FORCE_INLINE(VBOXSTRICTRC)
812	iemRegEip32RelativeJumpS8FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
813	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
814	{
815	Assert(!IEM_IS_64BIT_CODE(pVCpu));
816	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
817
818	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
819	if (enmEffOpSize == IEMMODE_16BIT)
820	uNewEip &= UINT16_MAX;
821	pVCpu->cpum.GstCtx.rip = uNewEip;
822
823	#ifndef IEM_WITH_CODE_TLB
824	iemOpcodeFlushLight(pVCpu, cbInstr);
825	#endif
826
827	/*
828	* Clear RF and finish the instruction (maybe raise #DB).
829	*/
830	return iemRegFinishClearingFlags(pVCpu, rcNormal);
831	}
832
833
834	/**
835	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
836	*
837	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
838	* segment limit.
839	*
840	* @param pVCpu The cross context virtual CPU structure of the calling thread.
841	* @param cbInstr Instruction size.
842	* @param offNextInstr The offset of the next instruction.
843	* @param rcNormal VINF_SUCCESS to continue TB.
844	* VINF_IEM_REEXEC_BREAK to force TB exit when
845	* taking the wrong conditional branhc.
846	*/
847	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
848	int8_t offNextInstr, int rcNormal) RT_NOEXCEPT
849	{
850	Assert(!IEM_IS_64BIT_CODE(pVCpu));
851
852	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
853	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
854	pVCpu->cpum.GstCtx.rip = uNewIp;
855	else
856	return iemRaiseGeneralProtectionFault0(pVCpu);
857
858	#ifndef IEM_WITH_CODE_TLB
859	iemOpcodeFlushLight(pVCpu, cbInstr);
860	#endif
861
862	/*
863	* Clear RF and finish the instruction (maybe raise #DB).
864	*/
865	return iemRegFinishClearingFlags(pVCpu, rcNormal);
866	}
867
868
869	/**
870	* Adds a 8-bit signed jump offset to RIP from 64-bit code, no checking or
871	* clearing of flags.
872	*
873	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
874	* segment limit.
875	*
876	* @param pVCpu The cross context virtual CPU structure of the calling thread.
877	* @param cbInstr Instruction size.
878	* @param offNextInstr The offset of the next instruction.
879	* @param enmEffOpSize Effective operand size.
880	* @param rcNormal VINF_SUCCESS to continue TB.
881	* VINF_IEM_REEXEC_BREAK to force TB exit when
882	* taking the wrong conditional branhc.
883	*/
884	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
885	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
886	{
887	Assert(IEM_IS_64BIT_CODE(pVCpu));
888	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
889
890	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
891	if (enmEffOpSize == IEMMODE_16BIT)
892	uNewRip &= UINT16_MAX;
893
894	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
895	pVCpu->cpum.GstCtx.rip = uNewRip;
896	else
897	return iemRaiseGeneralProtectionFault0(pVCpu);
898
899	#ifndef IEM_WITH_CODE_TLB
900	iemOpcodeFlushLight(pVCpu, cbInstr);
901	#endif
902	return iemRegFinishNoFlags(pVCpu, rcNormal);
903	}
904
905
906	/**
907	* Adds a 8-bit signed jump offset to RIP from 64-bit code when caller is sure
908	* it stays within the same page, no checking or clearing of flags.
909	*
910	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
911	* segment limit.
912	*
913	* @param pVCpu The cross context virtual CPU structure of the calling thread.
914	* @param cbInstr Instruction size.
915	* @param offNextInstr The offset of the next instruction.
916	* @param enmEffOpSize Effective operand size.
917	* @param rcNormal VINF_SUCCESS to continue TB.
918	* VINF_IEM_REEXEC_BREAK to force TB exit when
919	* taking the wrong conditional branhc.
920	*/
921	DECL_FORCE_INLINE(VBOXSTRICTRC)
922	iemRegRip64RelativeJumpS8IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
923	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
924	{
925	Assert(IEM_IS_64BIT_CODE(pVCpu));
926	Assert(enmEffOpSize == IEMMODE_64BIT); RT_NOREF(enmEffOpSize);
927
928	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
929	Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));
930	pVCpu->cpum.GstCtx.rip = uNewRip;
931
932	#ifndef IEM_WITH_CODE_TLB
933	iemOpcodeFlushLight(pVCpu, cbInstr);
934	#endif
935	return iemRegFinishNoFlags(pVCpu, rcNormal);
936	}
937
938
939	/**
940	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
941	* code (never 64-bit), no checking or clearing of flags.
942	*
943	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
944	* segment limit.
945	*
946	* @param pVCpu The cross context virtual CPU structure of the calling thread.
947	* @param cbInstr Instruction size.
948	* @param offNextInstr The offset of the next instruction.
949	* @param enmEffOpSize Effective operand size.
950	* @param rcNormal VINF_SUCCESS to continue TB.
951	* VINF_IEM_REEXEC_BREAK to force TB exit when
952	* taking the wrong conditional branhc.
953	*/
954	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
955	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
956	{
957	Assert(!IEM_IS_64BIT_CODE(pVCpu));
958	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
959
960	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
961	if (enmEffOpSize == IEMMODE_16BIT)
962	uNewEip &= UINT16_MAX;
963	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
964	pVCpu->cpum.GstCtx.rip = uNewEip;
965	else
966	return iemRaiseGeneralProtectionFault0(pVCpu);
967
968	#ifndef IEM_WITH_CODE_TLB
969	iemOpcodeFlushLight(pVCpu, cbInstr);
970	#endif
971	return iemRegFinishNoFlags(pVCpu, rcNormal);
972	}
973
974
975	/**
976	* Adds a 8-bit signed jump offset to EIP, on 386 or later from flat 32-bit code
977	* (never 64-bit), no checking or clearing of flags.
978	*
979	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
980	* segment limit.
981	*
982	* @param pVCpu The cross context virtual CPU structure of the calling thread.
983	* @param cbInstr Instruction size.
984	* @param offNextInstr The offset of the next instruction.
985	* @param enmEffOpSize Effective operand size.
986	* @param rcNormal VINF_SUCCESS to continue TB.
987	* VINF_IEM_REEXEC_BREAK to force TB exit when
988	* taking the wrong conditional branhc.
989	*/
990	DECL_FORCE_INLINE(VBOXSTRICTRC)
991	iemRegEip32RelativeJumpS8FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
992	IEMMODE enmEffOpSize, int rcNormal) RT_NOEXCEPT
993	{
994	Assert(!IEM_IS_64BIT_CODE(pVCpu));
995	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
996
997	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
998	if (enmEffOpSize == IEMMODE_16BIT)
999	uNewEip &= UINT16_MAX;
1000	pVCpu->cpum.GstCtx.rip = uNewEip;
1001
1002	#ifndef IEM_WITH_CODE_TLB
1003	iemOpcodeFlushLight(pVCpu, cbInstr);
1004	#endif
1005	return iemRegFinishNoFlags(pVCpu, rcNormal);
1006	}
1007
1008
1009	/**
1010	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU, no checking or
1011	* clearing of flags.
1012	*
1013	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1014	* segment limit.
1015	*
1016	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1017	* @param cbInstr Instruction size.
1018	* @param offNextInstr The offset of the next instruction.
1019	* @param rcNormal VINF_SUCCESS to continue TB.
1020	* VINF_IEM_REEXEC_BREAK to force TB exit when
1021	* taking the wrong conditional branhc.
1022	*/
1023	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1024	int8_t offNextInstr, int rcNormal) RT_NOEXCEPT
1025	{
1026	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1027
1028	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
1029	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
1030	pVCpu->cpum.GstCtx.rip = uNewIp;
1031	else
1032	return iemRaiseGeneralProtectionFault0(pVCpu);
1033
1034	#ifndef IEM_WITH_CODE_TLB
1035	iemOpcodeFlushLight(pVCpu, cbInstr);
1036	#endif
1037	return iemRegFinishNoFlags(pVCpu, rcNormal);
1038	}
1039
1040
1041	/**
1042	* Adds a 16-bit signed jump offset to RIP from 64-bit code.
1043	*
1044	* @returns Strict VBox status code.
1045	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1046	* @param cbInstr Instruction size.
1047	* @param offNextInstr The offset of the next instruction.
1048	* @param rcNormal VINF_SUCCESS to continue TB.
1049	* VINF_IEM_REEXEC_BREAK to force TB exit when
1050	* taking the wrong conditional branhc.
1051	*/
1052	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1053	int16_t offNextInstr, int rcNormal) RT_NOEXCEPT
1054	{
1055	Assert(IEM_IS_64BIT_CODE(pVCpu));
1056
1057	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
1058
1059	#ifndef IEM_WITH_CODE_TLB
1060	iemOpcodeFlushLight(pVCpu, cbInstr);
1061	#endif
1062
1063	/*
1064	* Clear RF and finish the instruction (maybe raise #DB).
1065	*/
1066	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1067	}
1068
1069
1070	/**
1071	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
1072	*
1073	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1074	* segment limit.
1075	*
1076	* @returns Strict VBox status code.
1077	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1078	* @param cbInstr Instruction size.
1079	* @param offNextInstr The offset of the next instruction.
1080	* @param rcNormal VINF_SUCCESS to continue TB.
1081	* VINF_IEM_REEXEC_BREAK to force TB exit when
1082	* taking the wrong conditional branhc.
1083	*
1084	* @note This is also used by 16-bit code in pre-386 mode, as the code is
1085	* identical.
1086	*/
1087	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1088	int16_t offNextInstr, int rcNormal) RT_NOEXCEPT
1089	{
1090	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1091
1092	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
1093	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
1094	pVCpu->cpum.GstCtx.rip = uNewIp;
1095	else
1096	return iemRaiseGeneralProtectionFault0(pVCpu);
1097
1098	#ifndef IEM_WITH_CODE_TLB
1099	iemOpcodeFlushLight(pVCpu, cbInstr);
1100	#endif
1101
1102	/*
1103	* Clear RF and finish the instruction (maybe raise #DB).
1104	*/
1105	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1106	}
1107
1108
1109	/**
1110	* Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code.
1111	*
1112	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1113	* segment limit.
1114	*
1115	* @returns Strict VBox status code.
1116	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1117	* @param cbInstr Instruction size.
1118	* @param offNextInstr The offset of the next instruction.
1119	* @param rcNormal VINF_SUCCESS to continue TB.
1120	* VINF_IEM_REEXEC_BREAK to force TB exit when
1121	* taking the wrong conditional branhc.
1122	*
1123	* @note This is also used by 16-bit code in pre-386 mode, as the code is
1124	* identical.
1125	*/
1126	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1127	int16_t offNextInstr, int rcNormal) RT_NOEXCEPT
1128	{
1129	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1130
1131	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
1132	pVCpu->cpum.GstCtx.rip = uNewIp;
1133
1134	#ifndef IEM_WITH_CODE_TLB
1135	iemOpcodeFlushLight(pVCpu, cbInstr);
1136	#endif
1137
1138	/*
1139	* Clear RF and finish the instruction (maybe raise #DB).
1140	*/
1141	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1142	}
1143
1144
1145	/**
1146	* Adds a 16-bit signed jump offset to RIP from 64-bit code, no checking or
1147	* clearing of flags.
1148	*
1149	* @returns Strict VBox status code.
1150	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1151	* @param cbInstr Instruction size.
1152	* @param offNextInstr The offset of the next instruction.
1153	* @param rcNormal VINF_SUCCESS to continue TB.
1154	* VINF_IEM_REEXEC_BREAK to force TB exit when
1155	* taking the wrong conditional branhc.
1156	*/
1157	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1158	int16_t offNextInstr, int rcNormal) RT_NOEXCEPT
1159	{
1160	Assert(IEM_IS_64BIT_CODE(pVCpu));
1161
1162	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
1163
1164	#ifndef IEM_WITH_CODE_TLB
1165	iemOpcodeFlushLight(pVCpu, cbInstr);
1166	#endif
1167	return iemRegFinishNoFlags(pVCpu, rcNormal);
1168	}
1169
1170
1171	/**
1172	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code,
1173	* no checking or clearing of flags.
1174	*
1175	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1176	* segment limit.
1177	*
1178	* @returns Strict VBox status code.
1179	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1180	* @param cbInstr Instruction size.
1181	* @param offNextInstr The offset of the next instruction.
1182	* @param rcNormal VINF_SUCCESS to continue TB.
1183	* VINF_IEM_REEXEC_BREAK to force TB exit when
1184	* taking the wrong conditional branhc.
1185	*
1186	* @note This is also used by 16-bit code in pre-386 mode, as the code is
1187	* identical.
1188	*/
1189	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1190	int16_t offNextInstr, int rcNormal) RT_NOEXCEPT
1191	{
1192	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1193
1194	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
1195	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
1196	pVCpu->cpum.GstCtx.rip = uNewIp;
1197	else
1198	return iemRaiseGeneralProtectionFault0(pVCpu);
1199
1200	#ifndef IEM_WITH_CODE_TLB
1201	iemOpcodeFlushLight(pVCpu, cbInstr);
1202	#endif
1203	return iemRegFinishNoFlags(pVCpu, rcNormal);
1204	}
1205
1206
1207	/**
1208	* Adds a 16-bit signed jump offset to EIP from FLAT 32-bit code, no checking or
1209	* clearing of flags.
1210	*
1211	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1212	* segment limit.
1213	*
1214	* @returns Strict VBox status code.
1215	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1216	* @param cbInstr Instruction size.
1217	* @param offNextInstr The offset of the next instruction.
1218	* @param rcNormal VINF_SUCCESS to continue TB.
1219	* VINF_IEM_REEXEC_BREAK to force TB exit when
1220	* taking the wrong conditional branhc.
1221	*
1222	* @note This is also used by 16-bit code in pre-386 mode, as the code is
1223	* identical.
1224	*/
1225	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1226	int16_t offNextInstr, int rcNormal) RT_NOEXCEPT
1227	{
1228	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1229
1230	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
1231	pVCpu->cpum.GstCtx.rip = uNewIp;
1232
1233	#ifndef IEM_WITH_CODE_TLB
1234	iemOpcodeFlushLight(pVCpu, cbInstr);
1235	#endif
1236	return iemRegFinishNoFlags(pVCpu, rcNormal);
1237	}
1238
1239
1240	/**
1241	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
1242	*
1243	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1244	* segment limit.
1245	*
1246	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
1247	* only alternative for relative jumps in 64-bit code and that is already
1248	* handled in the decoder stage.
1249	*
1250	* @returns Strict VBox status code.
1251	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1252	* @param cbInstr Instruction size.
1253	* @param offNextInstr The offset of the next instruction.
1254	* @param rcNormal VINF_SUCCESS to continue TB.
1255	* VINF_IEM_REEXEC_BREAK to force TB exit when
1256	* taking the wrong conditional branhc.
1257	*/
1258	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1259	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1260	{
1261	Assert(IEM_IS_64BIT_CODE(pVCpu));
1262
1263	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
1264	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
1265	pVCpu->cpum.GstCtx.rip = uNewRip;
1266	else
1267	return iemRaiseGeneralProtectionFault0(pVCpu);
1268
1269	#ifndef IEM_WITH_CODE_TLB
1270	iemOpcodeFlushLight(pVCpu, cbInstr);
1271	#endif
1272
1273	/*
1274	* Clear RF and finish the instruction (maybe raise #DB).
1275	*/
1276	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1277	}
1278
1279
1280	/**
1281	* Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is
1282	* sure the target is in the same page.
1283	*
1284	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1285	* segment limit.
1286	*
1287	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
1288	* only alternative for relative jumps in 64-bit code and that is already
1289	* handled in the decoder stage.
1290	*
1291	* @returns Strict VBox status code.
1292	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1293	* @param cbInstr Instruction size.
1294	* @param offNextInstr The offset of the next instruction.
1295	* @param rcNormal VINF_SUCCESS to continue TB.
1296	* VINF_IEM_REEXEC_BREAK to force TB exit when
1297	* taking the wrong conditional branhc.
1298	*/
1299	DECL_FORCE_INLINE(VBOXSTRICTRC)
1300	iemRegRip64RelativeJumpS32IntraPgAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1301	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1302	{
1303	Assert(IEM_IS_64BIT_CODE(pVCpu));
1304
1305	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
1306	Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));
1307	pVCpu->cpum.GstCtx.rip = uNewRip;
1308
1309	#ifndef IEM_WITH_CODE_TLB
1310	iemOpcodeFlushLight(pVCpu, cbInstr);
1311	#endif
1312
1313	/*
1314	* Clear RF and finish the instruction (maybe raise #DB).
1315	*/
1316	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1317	}
1318
1319
1320	/**
1321	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
1322	*
1323	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1324	* segment limit.
1325	*
1326	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
1327	* only alternative for relative jumps in 32-bit code and that is already
1328	* handled in the decoder stage.
1329	*
1330	* @returns Strict VBox status code.
1331	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1332	* @param cbInstr Instruction size.
1333	* @param offNextInstr The offset of the next instruction.
1334	* @param rcNormal VINF_SUCCESS to continue TB.
1335	* VINF_IEM_REEXEC_BREAK to force TB exit when
1336	* taking the wrong conditional branhc.
1337	*/
1338	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1339	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1340	{
1341	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1342	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1343
1344	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
1345	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
1346	pVCpu->cpum.GstCtx.rip = uNewEip;
1347	else
1348	return iemRaiseGeneralProtectionFault0(pVCpu);
1349
1350	#ifndef IEM_WITH_CODE_TLB
1351	iemOpcodeFlushLight(pVCpu, cbInstr);
1352	#endif
1353
1354	/*
1355	* Clear RF and finish the instruction (maybe raise #DB).
1356	*/
1357	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1358	}
1359
1360
1361	/**
1362	* Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code.
1363	*
1364	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1365	* segment limit.
1366	*
1367	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
1368	* only alternative for relative jumps in 32-bit code and that is already
1369	* handled in the decoder stage.
1370	*
1371	* @returns Strict VBox status code.
1372	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1373	* @param cbInstr Instruction size.
1374	* @param offNextInstr The offset of the next instruction.
1375	* @param rcNormal VINF_SUCCESS to continue TB.
1376	* VINF_IEM_REEXEC_BREAK to force TB exit when
1377	* taking the wrong conditional branhc.
1378	*/
1379	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
1380	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1381	{
1382	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1383	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1384
1385	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
1386	pVCpu->cpum.GstCtx.rip = uNewEip;
1387
1388	#ifndef IEM_WITH_CODE_TLB
1389	iemOpcodeFlushLight(pVCpu, cbInstr);
1390	#endif
1391
1392	/*
1393	* Clear RF and finish the instruction (maybe raise #DB).
1394	*/
1395	return iemRegFinishClearingFlags(pVCpu, rcNormal);
1396	}
1397
1398
1399
1400	/**
1401	* Adds a 32-bit signed jump offset to RIP from 64-bit code, no checking or
1402	* clearing of flags.
1403	*
1404	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1405	* segment limit.
1406	*
1407	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
1408	* only alternative for relative jumps in 64-bit code and that is already
1409	* handled in the decoder stage.
1410	*
1411	* @returns Strict VBox status code.
1412	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1413	* @param cbInstr Instruction size.
1414	* @param offNextInstr The offset of the next instruction.
1415	* @param rcNormal VINF_SUCCESS to continue TB.
1416	* VINF_IEM_REEXEC_BREAK to force TB exit when
1417	* taking the wrong conditional branhc.
1418	*/
1419	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1420	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1421	{
1422	Assert(IEM_IS_64BIT_CODE(pVCpu));
1423
1424	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
1425	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
1426	pVCpu->cpum.GstCtx.rip = uNewRip;
1427	else
1428	return iemRaiseGeneralProtectionFault0(pVCpu);
1429
1430	#ifndef IEM_WITH_CODE_TLB
1431	iemOpcodeFlushLight(pVCpu, cbInstr);
1432	#endif
1433	return iemRegFinishNoFlags(pVCpu, rcNormal);
1434	}
1435
1436
1437	/**
1438	* Adds a 32-bit signed jump offset to RIP from 64-bit code when the caller is
1439	* sure it stays within the same page, no checking or clearing of flags.
1440	*
1441	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1442	* segment limit.
1443	*
1444	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
1445	* only alternative for relative jumps in 64-bit code and that is already
1446	* handled in the decoder stage.
1447	*
1448	* @returns Strict VBox status code.
1449	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1450	* @param cbInstr Instruction size.
1451	* @param offNextInstr The offset of the next instruction.
1452	* @param rcNormal VINF_SUCCESS to continue TB.
1453	* VINF_IEM_REEXEC_BREAK to force TB exit when
1454	* taking the wrong conditional branhc.
1455	*/
1456	DECL_FORCE_INLINE(VBOXSTRICTRC)
1457	iemRegRip64RelativeJumpS32IntraPgAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1458	{
1459	Assert(IEM_IS_64BIT_CODE(pVCpu));
1460
1461	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
1462	Assert((pVCpu->cpum.GstCtx.rip >> GUEST_PAGE_SHIFT) == (uNewRip >> GUEST_PAGE_SHIFT));
1463	pVCpu->cpum.GstCtx.rip = uNewRip;
1464
1465	#ifndef IEM_WITH_CODE_TLB
1466	iemOpcodeFlushLight(pVCpu, cbInstr);
1467	#endif
1468	return iemRegFinishNoFlags(pVCpu, rcNormal);
1469	}
1470
1471
1472	/**
1473	* Adds a 32-bit signed jump offset to RIP from 32-bit code, no checking or
1474	* clearing of flags.
1475	*
1476	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1477	* segment limit.
1478	*
1479	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
1480	* only alternative for relative jumps in 32-bit code and that is already
1481	* handled in the decoder stage.
1482	*
1483	* @returns Strict VBox status code.
1484	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1485	* @param cbInstr Instruction size.
1486	* @param offNextInstr The offset of the next instruction.
1487	* @param rcNormal VINF_SUCCESS to continue TB.
1488	* VINF_IEM_REEXEC_BREAK to force TB exit when
1489	* taking the wrong conditional branhc.
1490	*/
1491	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1492	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1493	{
1494	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1495	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1496
1497	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
1498	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
1499	pVCpu->cpum.GstCtx.rip = uNewEip;
1500	else
1501	return iemRaiseGeneralProtectionFault0(pVCpu);
1502
1503	#ifndef IEM_WITH_CODE_TLB
1504	iemOpcodeFlushLight(pVCpu, cbInstr);
1505	#endif
1506	return iemRegFinishNoFlags(pVCpu, rcNormal);
1507	}
1508
1509
1510	/**
1511	* Adds a 32-bit signed jump offset to RIP from FLAT 32-bit code, no checking or
1512	* clearing of flags.
1513	*
1514	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1515	* segment limit.
1516	*
1517	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
1518	* only alternative for relative jumps in 32-bit code and that is already
1519	* handled in the decoder stage.
1520	*
1521	* @returns Strict VBox status code.
1522	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1523	* @param cbInstr Instruction size.
1524	* @param offNextInstr The offset of the next instruction.
1525	* @param rcNormal VINF_SUCCESS to continue TB.
1526	* VINF_IEM_REEXEC_BREAK to force TB exit when
1527	* taking the wrong conditional branhc.
1528	*/
1529	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32FlatAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr,
1530	int32_t offNextInstr, int rcNormal) RT_NOEXCEPT
1531	{
1532	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1533	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1534
1535	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
1536	pVCpu->cpum.GstCtx.rip = uNewEip;
1537
1538	#ifndef IEM_WITH_CODE_TLB
1539	iemOpcodeFlushLight(pVCpu, cbInstr);
1540	#endif
1541	return iemRegFinishNoFlags(pVCpu, rcNormal);
1542	}
1543
1544
1545	/**
1546	* Extended version of iemFinishInstructionWithFlagsSet that goes with
1547	* iemRegAddToRipAndFinishingClearingRfEx.
1548	*
1549	* See iemFinishInstructionWithFlagsSet() for details.
1550	*/
1551	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
1552	{
1553	/*
1554	* Raise a #DB.
1555	*/
1556	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
1557	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
1558	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
1559	\| ( (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK_NONSILENT)
1560	>> CPUMCTX_DBG_HIT_DRX_SHIFT);
1561	/** @todo Do we set all pending \#DB events, or just one? */
1562	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
1563	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
1564	pVCpu->cpum.GstCtx.rflags.uBoth));
1565	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
1566	return iemRaiseDebugException(pVCpu);
1567	}
1568
1569
1570	/**
1571	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
1572	* others potentially updating EFLAGS.TF.
1573	*
1574	* The single step event must be generated using the TF value at the start of
1575	* the instruction, not the new value set by it.
1576	*
1577	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1578	* @param cbInstr The number of bytes to add.
1579	* @param fEflOld The EFLAGS at the start of the instruction
1580	* execution.
1581	*/
1582	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
1583	{
1584	iemRegAddToRip(pVCpu, cbInstr);
1585	if (!(fEflOld & X86_EFL_TF))
1586	{
1587	/* Specialized iemRegFinishClearingFlags edition here that doesn't check X86_EFL_TF. */
1588	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
1589	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
1590	& (X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
1591	return VINF_SUCCESS;
1592	return iemFinishInstructionWithFlagsSet<0 /a_fTF/>(pVCpu, VINF_SUCCESS); /* TF=0, so ignore it. */
1593	}
1594	return iemFinishInstructionWithTfSet(pVCpu);
1595	}
1596
1597
1598	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1599	/**
1600	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1601	*
1602	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1603	*/
1604	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
1605	{
1606	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1607	}
1608	#endif
1609
1610
1611	#ifdef IEM_WITH_CODE_TLB
1612
1613	/**
1614	* Performs a near jump to the specified address, no checking or clearing of
1615	* flags
1616	*
1617	* May raise a \#GP(0) if the new IP outside the code segment limit.
1618	*
1619	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1620	* @param uNewIp The new IP value.
1621	*/
1622	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishNoFlags(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT
1623	{
1624	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
1625	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))
1626	pVCpu->cpum.GstCtx.rip = uNewIp;
1627	else
1628	return iemRaiseGeneralProtectionFault0(pVCpu);
1629	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1630	}
1631
1632
1633	/**
1634	* Performs a near jump to the specified address, no checking or clearing of
1635	* flags
1636	*
1637	* May raise a \#GP(0) if the new RIP is outside the code segment limit.
1638	*
1639	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1640	* @param uNewEip The new EIP value.
1641	*/
1642	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishNoFlags(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT
1643	{
1644	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1645	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1646	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
1647	pVCpu->cpum.GstCtx.rip = uNewEip;
1648	else
1649	return iemRaiseGeneralProtectionFault0(pVCpu);
1650	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1651	}
1652
1653
1654	/**
1655	* Performs a near jump to the specified address, no checking or clearing of
1656	* flags.
1657	*
1658	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1659	* segment limit.
1660	*
1661	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1662	* @param uNewRip The new RIP value.
1663	*/
1664	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishNoFlags(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
1665	{
1666	Assert(IEM_IS_64BIT_CODE(pVCpu));
1667	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
1668	pVCpu->cpum.GstCtx.rip = uNewRip;
1669	else
1670	return iemRaiseGeneralProtectionFault0(pVCpu);
1671	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1672	}
1673
1674	#endif /* IEM_WITH_CODE_TLB */
1675
1676	/**
1677	* Performs a near jump to the specified address.
1678	*
1679	* May raise a \#GP(0) if the new IP outside the code segment limit.
1680	*
1681	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1682	* @param uNewIp The new IP value.
1683	* @param cbInstr The instruction length, for flushing in the non-TLB case.
1684	*/
1685	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU16AndFinishClearingRF(PVMCPUCC pVCpu, uint16_t uNewIp, uint8_t cbInstr) RT_NOEXCEPT
1686	{
1687	if (RT_LIKELY( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
1688	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no limit checks in 64-bit mode */))
1689	pVCpu->cpum.GstCtx.rip = uNewIp;
1690	else
1691	return iemRaiseGeneralProtectionFault0(pVCpu);
1692	#ifndef IEM_WITH_CODE_TLB
1693	iemOpcodeFlushLight(pVCpu, cbInstr);
1694	#else
1695	RT_NOREF_PV(cbInstr);
1696	#endif
1697	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
1698	}
1699
1700
1701	/**
1702	* Performs a near jump to the specified address.
1703	*
1704	* May raise a \#GP(0) if the new RIP is outside the code segment limit.
1705	*
1706	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1707	* @param uNewEip The new EIP value.
1708	* @param cbInstr The instruction length, for flushing in the non-TLB case.
1709	*/
1710	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU32AndFinishClearingRF(PVMCPUCC pVCpu, uint32_t uNewEip, uint8_t cbInstr) RT_NOEXCEPT
1711	{
1712	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1713	Assert(!IEM_IS_64BIT_CODE(pVCpu));
1714	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
1715	pVCpu->cpum.GstCtx.rip = uNewEip;
1716	else
1717	return iemRaiseGeneralProtectionFault0(pVCpu);
1718	#ifndef IEM_WITH_CODE_TLB
1719	iemOpcodeFlushLight(pVCpu, cbInstr);
1720	#else
1721	RT_NOREF_PV(cbInstr);
1722	#endif
1723	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
1724	}
1725
1726
1727	/**
1728	* Performs a near jump to the specified address.
1729	*
1730	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
1731	* segment limit.
1732	*
1733	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1734	* @param uNewRip The new RIP value.
1735	* @param cbInstr The instruction length, for flushing in the non-TLB case.
1736	*/
1737	DECLINLINE(VBOXSTRICTRC) iemRegRipJumpU64AndFinishClearingRF(PVMCPUCC pVCpu, uint64_t uNewRip, uint8_t cbInstr) RT_NOEXCEPT
1738	{
1739	Assert(IEM_IS_64BIT_CODE(pVCpu));
1740	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
1741	pVCpu->cpum.GstCtx.rip = uNewRip;
1742	else
1743	return iemRaiseGeneralProtectionFault0(pVCpu);
1744	#ifndef IEM_WITH_CODE_TLB
1745	iemOpcodeFlushLight(pVCpu, cbInstr);
1746	#else
1747	RT_NOREF_PV(cbInstr);
1748	#endif
1749	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
1750	}
1751
1752
1753	/**
1754	* Implements a 16-bit relative call, no checking or clearing of
1755	* flags.
1756	*
1757	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1758	* @param cbInstr The instruction length.
1759	* @param offDisp The 16-bit displacement.
1760	*/
1761	DECL_FORCE_INLINE(VBOXSTRICTRC)
1762	iemRegRipRelativeCallS16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT
1763	{
1764	uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr;
1765	uint16_t const uNewIp = uOldIp + offDisp;
1766	if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
1767	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */)
1768	{ /* likely */ }
1769	else
1770	return iemRaiseGeneralProtectionFault0(pVCpu);
1771
1772	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp);
1773	if (rcStrict == VINF_SUCCESS)
1774	{ /* likely */ }
1775	else
1776	return rcStrict;
1777
1778	pVCpu->cpum.GstCtx.rip = uNewIp;
1779	#ifndef IEM_WITH_CODE_TLB
1780	iemOpcodeFlushLight(pVCpu, cbInstr);
1781	#endif
1782	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1783	}
1784
1785
1786	/**
1787	* Implements a 16-bit relative call.
1788	*
1789	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1790	* @param cbInstr The instruction length.
1791	* @param offDisp The 16-bit displacement.
1792	*/
1793	DECL_FORCE_INLINE(VBOXSTRICTRC)
1794	iemRegRipRelativeCallS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offDisp) RT_NOEXCEPT
1795	{
1796	uint16_t const uOldIp = pVCpu->cpum.GstCtx.ip + cbInstr;
1797	uint16_t const uNewIp = uOldIp + offDisp;
1798	if ( uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
1799	\|\| IEM_IS_64BIT_CODE(pVCpu) /* no CS limit checks in 64-bit mode */)
1800	{ /* likely */ }
1801	else
1802	return iemRaiseGeneralProtectionFault0(pVCpu);
1803
1804	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldIp);
1805	if (rcStrict == VINF_SUCCESS)
1806	{ /* likely */ }
1807	else
1808	return rcStrict;
1809
1810	pVCpu->cpum.GstCtx.rip = uNewIp;
1811	#ifndef IEM_WITH_CODE_TLB
1812	iemOpcodeFlushLight(pVCpu, cbInstr);
1813	#endif
1814	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
1815	}
1816
1817
1818	/**
1819	* Implements a 32-bit relative call, no checking or clearing of flags.
1820	*
1821	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1822	* @param cbInstr The instruction length.
1823	* @param offDisp The 32-bit displacement.
1824	*/
1825	DECL_FORCE_INLINE(VBOXSTRICTRC)
1826	iemRegEip32RelativeCallS32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT
1827	{
1828	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));
1829
1830	uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;
1831	uint32_t const uNewRip = uOldRip + offDisp;
1832	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
1833	{ /* likely */ }
1834	else
1835	return iemRaiseGeneralProtectionFault0(pVCpu);
1836
1837	VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);
1838	if (rcStrict == VINF_SUCCESS)
1839	{ /* likely */ }
1840	else
1841	return rcStrict;
1842
1843	pVCpu->cpum.GstCtx.rip = uNewRip;
1844	#ifndef IEM_WITH_CODE_TLB
1845	iemOpcodeFlushLight(pVCpu, cbInstr);
1846	#endif
1847	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1848	}
1849
1850
1851	/**
1852	* Implements a 32-bit relative call.
1853	*
1854	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1855	* @param cbInstr The instruction length.
1856	* @param offDisp The 32-bit displacement.
1857	*/
1858	DECL_FORCE_INLINE(VBOXSTRICTRC)
1859	iemRegEip32RelativeCallS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offDisp) RT_NOEXCEPT
1860	{
1861	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(!IEM_IS_64BIT_CODE(pVCpu));
1862
1863	uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;
1864	uint32_t const uNewRip = uOldRip + offDisp;
1865	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
1866	{ /* likely */ }
1867	else
1868	return iemRaiseGeneralProtectionFault0(pVCpu);
1869
1870	VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);
1871	if (rcStrict == VINF_SUCCESS)
1872	{ /* likely */ }
1873	else
1874	return rcStrict;
1875
1876	pVCpu->cpum.GstCtx.rip = uNewRip;
1877	#ifndef IEM_WITH_CODE_TLB
1878	iemOpcodeFlushLight(pVCpu, cbInstr);
1879	#endif
1880	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
1881	}
1882
1883
1884	/**
1885	* Implements a 64-bit relative call, no checking or clearing of flags.
1886	*
1887	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1888	* @param cbInstr The instruction length.
1889	* @param offDisp The 64-bit displacement.
1890	*/
1891	DECL_FORCE_INLINE(VBOXSTRICTRC)
1892	iemRegRip64RelativeCallS64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT
1893	{
1894	uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;
1895	uint64_t const uNewRip = uOldRip + (int64_t)offDisp;
1896	if (IEM_IS_CANONICAL(uNewRip))
1897	{ /* likely */ }
1898	else
1899	return iemRaiseNotCanonical(pVCpu);
1900
1901	VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);
1902	if (rcStrict == VINF_SUCCESS)
1903	{ /* likely */ }
1904	else
1905	return rcStrict;
1906
1907	pVCpu->cpum.GstCtx.rip = uNewRip;
1908	#ifndef IEM_WITH_CODE_TLB
1909	iemOpcodeFlushLight(pVCpu, cbInstr);
1910	#endif
1911	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1912	}
1913
1914
1915	/**
1916	* Implements a 64-bit relative call.
1917	*
1918	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1919	* @param cbInstr The instruction length.
1920	* @param offDisp The 64-bit displacement.
1921	*/
1922	DECL_FORCE_INLINE(VBOXSTRICTRC)
1923	iemRegRip64RelativeCallS64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int64_t offDisp) RT_NOEXCEPT
1924	{
1925	uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;
1926	uint64_t const uNewRip = uOldRip + (int64_t)offDisp;
1927	if (IEM_IS_CANONICAL(uNewRip))
1928	{ /* likely */ }
1929	else
1930	return iemRaiseNotCanonical(pVCpu);
1931
1932	VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);
1933	if (rcStrict == VINF_SUCCESS)
1934	{ /* likely */ }
1935	else
1936	return rcStrict;
1937
1938	pVCpu->cpum.GstCtx.rip = uNewRip;
1939	#ifndef IEM_WITH_CODE_TLB
1940	iemOpcodeFlushLight(pVCpu, cbInstr);
1941	#endif
1942	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
1943	}
1944
1945
1946	/**
1947	* Implements an 16-bit indirect call, no checking or clearing of
1948	* flags.
1949	*
1950	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1951	* @param cbInstr The instruction length.
1952	* @param uNewRip The new RIP value.
1953	*/
1954	DECL_FORCE_INLINE(VBOXSTRICTRC)
1955	iemRegIp16IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT
1956	{
1957	uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;
1958	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
1959	{ /* likely */ }
1960	else
1961	return iemRaiseGeneralProtectionFault0(pVCpu);
1962
1963	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);
1964	if (rcStrict == VINF_SUCCESS)
1965	{ /* likely */ }
1966	else
1967	return rcStrict;
1968
1969	pVCpu->cpum.GstCtx.rip = uNewRip;
1970	#ifndef IEM_WITH_CODE_TLB
1971	iemOpcodeFlushLight(pVCpu, cbInstr);
1972	#endif
1973	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
1974	}
1975
1976
1977	/**
1978	* Implements an 16-bit indirect call, no checking or clearing of
1979	* flags.
1980	*
1981	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1982	* @param cbInstr The instruction length.
1983	* @param uNewRip The new RIP value.
1984	*/
1985	DECL_FORCE_INLINE(VBOXSTRICTRC)
1986	iemRegEip32IndirectCallU16AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT
1987	{
1988	uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;
1989	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
1990	{ /* likely */ }
1991	else
1992	return iemRaiseGeneralProtectionFault0(pVCpu);
1993
1994	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);
1995	if (rcStrict == VINF_SUCCESS)
1996	{ /* likely */ }
1997	else
1998	return rcStrict;
1999
2000	pVCpu->cpum.GstCtx.rip = uNewRip;
2001	#ifndef IEM_WITH_CODE_TLB
2002	iemOpcodeFlushLight(pVCpu, cbInstr);
2003	#endif
2004	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
2005	}
2006
2007
2008	/**
2009	* Implements an 16-bit indirect call.
2010	*
2011	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2012	* @param cbInstr The instruction length.
2013	* @param uNewRip The new RIP value.
2014	*/
2015	DECL_FORCE_INLINE(VBOXSTRICTRC)
2016	iemRegIp16IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT
2017	{
2018	uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;
2019	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
2020	{ /* likely */ }
2021	else
2022	return iemRaiseGeneralProtectionFault0(pVCpu);
2023
2024	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);
2025	if (rcStrict == VINF_SUCCESS)
2026	{ /* likely */ }
2027	else
2028	return rcStrict;
2029
2030	pVCpu->cpum.GstCtx.rip = uNewRip;
2031	#ifndef IEM_WITH_CODE_TLB
2032	iemOpcodeFlushLight(pVCpu, cbInstr);
2033	#endif
2034	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
2035	}
2036
2037
2038	/**
2039	* Implements an 16-bit indirect call.
2040	*
2041	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2042	* @param cbInstr The instruction length.
2043	* @param uNewRip The new RIP value.
2044	*/
2045	DECL_FORCE_INLINE(VBOXSTRICTRC)
2046	iemRegEip32IndirectCallU16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uNewRip) RT_NOEXCEPT
2047	{
2048	uint16_t const uOldRip = pVCpu->cpum.GstCtx.ip + cbInstr;
2049	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
2050	{ /* likely */ }
2051	else
2052	return iemRaiseGeneralProtectionFault0(pVCpu);
2053
2054	VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldRip);
2055	if (rcStrict == VINF_SUCCESS)
2056	{ /* likely */ }
2057	else
2058	return rcStrict;
2059
2060	pVCpu->cpum.GstCtx.rip = uNewRip;
2061	#ifndef IEM_WITH_CODE_TLB
2062	iemOpcodeFlushLight(pVCpu, cbInstr);
2063	#endif
2064	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
2065	}
2066
2067
2068	/**
2069	* Implements an 32-bit indirect call, no checking or clearing of
2070	* flags.
2071	*
2072	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2073	* @param cbInstr The instruction length.
2074	* @param uNewRip The new RIP value.
2075	*/
2076	DECL_FORCE_INLINE(VBOXSTRICTRC)
2077	iemRegEip32IndirectCallU32AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT
2078	{
2079	uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;
2080	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
2081	{ /* likely */ }
2082	else
2083	return iemRaiseGeneralProtectionFault0(pVCpu);
2084
2085	VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);
2086	if (rcStrict == VINF_SUCCESS)
2087	{ /* likely */ }
2088	else
2089	return rcStrict;
2090
2091	pVCpu->cpum.GstCtx.rip = uNewRip;
2092	#ifndef IEM_WITH_CODE_TLB
2093	iemOpcodeFlushLight(pVCpu, cbInstr);
2094	#endif
2095	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
2096	}
2097
2098
2099	/**
2100	* Implements an 32-bit indirect call.
2101	*
2102	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2103	* @param cbInstr The instruction length.
2104	* @param uNewRip The new RIP value.
2105	*/
2106	DECL_FORCE_INLINE(VBOXSTRICTRC)
2107	iemRegEip32IndirectCallU32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t uNewRip) RT_NOEXCEPT
2108	{
2109	uint32_t const uOldRip = pVCpu->cpum.GstCtx.eip + cbInstr;
2110	if (uNewRip <= pVCpu->cpum.GstCtx.cs.u32Limit)
2111	{ /* likely */ }
2112	else
2113	return iemRaiseGeneralProtectionFault0(pVCpu);
2114
2115	VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldRip);
2116	if (rcStrict == VINF_SUCCESS)
2117	{ /* likely */ }
2118	else
2119	return rcStrict;
2120
2121	pVCpu->cpum.GstCtx.rip = uNewRip;
2122	#ifndef IEM_WITH_CODE_TLB
2123	iemOpcodeFlushLight(pVCpu, cbInstr);
2124	#endif
2125	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
2126	}
2127
2128
2129	/**
2130	* Implements an 64-bit indirect call, no checking or clearing of
2131	* flags.
2132	*
2133	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2134	* @param cbInstr The instruction length.
2135	* @param uNewRip The new RIP value.
2136	*/
2137	DECL_FORCE_INLINE(VBOXSTRICTRC)
2138	iemRegRip64IndirectCallU64AndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT
2139	{
2140	uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;
2141	if (IEM_IS_CANONICAL(uNewRip))
2142	{ /* likely */ }
2143	else
2144	return iemRaiseGeneralProtectionFault0(pVCpu);
2145
2146	VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);
2147	if (rcStrict == VINF_SUCCESS)
2148	{ /* likely */ }
2149	else
2150	return rcStrict;
2151
2152	pVCpu->cpum.GstCtx.rip = uNewRip;
2153	#ifndef IEM_WITH_CODE_TLB
2154	iemOpcodeFlushLight(pVCpu, cbInstr);
2155	#endif
2156	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
2157	}
2158
2159
2160	/**
2161	* Implements an 64-bit indirect call.
2162	*
2163	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2164	* @param cbInstr The instruction length.
2165	* @param uNewRip The new RIP value.
2166	*/
2167	DECL_FORCE_INLINE(VBOXSTRICTRC)
2168	iemRegRip64IndirectCallU64AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t uNewRip) RT_NOEXCEPT
2169	{
2170	uint64_t const uOldRip = pVCpu->cpum.GstCtx.rip + cbInstr;
2171	if (IEM_IS_CANONICAL(uNewRip))
2172	{ /* likely */ }
2173	else
2174	return iemRaiseGeneralProtectionFault0(pVCpu);
2175
2176	VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldRip);
2177	if (rcStrict == VINF_SUCCESS)
2178	{ /* likely */ }
2179	else
2180	return rcStrict;
2181
2182	pVCpu->cpum.GstCtx.rip = uNewRip;
2183	#ifndef IEM_WITH_CODE_TLB
2184	iemOpcodeFlushLight(pVCpu, cbInstr);
2185	#endif
2186	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
2187	}
2188
2189
2190
2191	/**
2192	* Adds to the stack pointer.
2193	*
2194	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2195	* @param cbToAdd The number of bytes to add (8-bit!).
2196	*/
2197	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2198	{
2199	if (IEM_IS_64BIT_CODE(pVCpu))
2200	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2201	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2202	pVCpu->cpum.GstCtx.esp += cbToAdd;
2203	else
2204	pVCpu->cpum.GstCtx.sp += cbToAdd;
2205	}
2206
2207
2208	/**
2209	* Subtracts from the stack pointer.
2210	*
2211	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2212	* @param cbToSub The number of bytes to subtract (8-bit!).
2213	*/
2214	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2215	{
2216	if (IEM_IS_64BIT_CODE(pVCpu))
2217	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2218	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2219	pVCpu->cpum.GstCtx.esp -= cbToSub;
2220	else
2221	pVCpu->cpum.GstCtx.sp -= cbToSub;
2222	}
2223
2224
2225	/**
2226	* Adds to the temporary stack pointer.
2227	*
2228	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2229	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2230	* @param cbToAdd The number of bytes to add (16-bit).
2231	*/
2232	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2233	{
2234	if (IEM_IS_64BIT_CODE(pVCpu))
2235	pTmpRsp->u += cbToAdd;
2236	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2237	pTmpRsp->DWords.dw0 += cbToAdd;
2238	else
2239	pTmpRsp->Words.w0 += cbToAdd;
2240	}
2241
2242
2243	/**
2244	* Subtracts from the temporary stack pointer.
2245	*
2246	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2247	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2248	* @param cbToSub The number of bytes to subtract.
2249	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2250	* expecting that.
2251	*/
2252	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2253	{
2254	if (IEM_IS_64BIT_CODE(pVCpu))
2255	pTmpRsp->u -= cbToSub;
2256	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2257	pTmpRsp->DWords.dw0 -= cbToSub;
2258	else
2259	pTmpRsp->Words.w0 -= cbToSub;
2260	}
2261
2262
2263	/**
2264	* Calculates the effective stack address for a push of the specified size as
2265	* well as the new RSP value (upper bits may be masked).
2266	*
2267	* @returns Effective stack addressf for the push.
2268	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2269	* @param cbItem The size of the stack item to pop.
2270	* @param puNewRsp Where to return the new RSP value.
2271	*/
2272	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2273	{
2274	RTUINT64U uTmpRsp;
2275	RTGCPTR GCPtrTop;
2276	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2277
2278	if (IEM_IS_64BIT_CODE(pVCpu))
2279	GCPtrTop = uTmpRsp.u -= cbItem;
2280	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2281	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2282	else
2283	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2284	*puNewRsp = uTmpRsp.u;
2285	return GCPtrTop;
2286	}
2287
2288
2289	/**
2290	* Gets the current stack pointer and calculates the value after a pop of the
2291	* specified size.
2292	*
2293	* @returns Current stack pointer.
2294	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2295	* @param cbItem The size of the stack item to pop.
2296	* @param puNewRsp Where to return the new RSP value.
2297	*/
2298	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2299	{
2300	RTUINT64U uTmpRsp;
2301	RTGCPTR GCPtrTop;
2302	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2303
2304	if (IEM_IS_64BIT_CODE(pVCpu))
2305	{
2306	GCPtrTop = uTmpRsp.u;
2307	uTmpRsp.u += cbItem;
2308	}
2309	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2310	{
2311	GCPtrTop = uTmpRsp.DWords.dw0;
2312	uTmpRsp.DWords.dw0 += cbItem;
2313	}
2314	else
2315	{
2316	GCPtrTop = uTmpRsp.Words.w0;
2317	uTmpRsp.Words.w0 += cbItem;
2318	}
2319	*puNewRsp = uTmpRsp.u;
2320	return GCPtrTop;
2321	}
2322
2323
2324	/**
2325	* Calculates the effective stack address for a push of the specified size as
2326	* well as the new temporary RSP value (upper bits may be masked).
2327	*
2328	* @returns Effective stack addressf for the push.
2329	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2330	* @param pTmpRsp The temporary stack pointer. This is updated.
2331	* @param cbItem The size of the stack item to pop.
2332	*/
2333	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2334	{
2335	RTGCPTR GCPtrTop;
2336
2337	if (IEM_IS_64BIT_CODE(pVCpu))
2338	GCPtrTop = pTmpRsp->u -= cbItem;
2339	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2340	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2341	else
2342	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2343	return GCPtrTop;
2344	}
2345
2346
2347	/**
2348	* Gets the effective stack address for a pop of the specified size and
2349	* calculates and updates the temporary RSP.
2350	*
2351	* @returns Current stack pointer.
2352	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2353	* @param pTmpRsp The temporary stack pointer. This is updated.
2354	* @param cbItem The size of the stack item to pop.
2355	*/
2356	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2357	{
2358	RTGCPTR GCPtrTop;
2359	if (IEM_IS_64BIT_CODE(pVCpu))
2360	{
2361	GCPtrTop = pTmpRsp->u;
2362	pTmpRsp->u += cbItem;
2363	}
2364	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2365	{
2366	GCPtrTop = pTmpRsp->DWords.dw0;
2367	pTmpRsp->DWords.dw0 += cbItem;
2368	}
2369	else
2370	{
2371	GCPtrTop = pTmpRsp->Words.w0;
2372	pTmpRsp->Words.w0 += cbItem;
2373	}
2374	return GCPtrTop;
2375	}
2376
2377
2378	/** Common body for iemRegRipNearReturnAndFinishClearingRF()
2379	* and iemRegRipNearReturnAndFinishNoFlags(). */
2380	template<bool a_fWithFlags>
2381	DECL_FORCE_INLINE(VBOXSTRICTRC)
2382	iemRegRipNearReturnCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT
2383	{
2384	/* Fetch the new RIP from the stack. */
2385	VBOXSTRICTRC rcStrict;
2386	RTUINT64U NewRip;
2387	RTUINT64U NewRsp;
2388	NewRsp.u = pVCpu->cpum.GstCtx.rsp;
2389	switch (enmEffOpSize)
2390	{
2391	case IEMMODE_16BIT:
2392	NewRip.u = 0;
2393	rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp);
2394	break;
2395	case IEMMODE_32BIT:
2396	NewRip.u = 0;
2397	rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp);
2398	break;
2399	case IEMMODE_64BIT:
2400	rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp);
2401	break;
2402	IEM_NOT_REACHED_DEFAULT_CASE_RET();
2403	}
2404	if (rcStrict != VINF_SUCCESS)
2405	return rcStrict;
2406
2407	/* Check the new ew RIP before loading it. */
2408	/** @todo Should test this as the intel+amd pseudo code doesn't mention half
2409	* of it. The canonical test is performed here and for call. */
2410	if (enmEffOpSize != IEMMODE_64BIT)
2411	{
2412	if (RT_LIKELY(NewRip.DWords.dw0 <= pVCpu->cpum.GstCtx.cs.u32Limit))
2413	{ /* likely */ }
2414	else
2415	{
2416	Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit));
2417	return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
2418	}
2419	}
2420	else
2421	{
2422	if (RT_LIKELY(IEM_IS_CANONICAL(NewRip.u)))
2423	{ /* likely */ }
2424	else
2425	{
2426	Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u));
2427	return iemRaiseNotCanonical(pVCpu);
2428	}
2429	}
2430
2431	/* Apply cbPop */
2432	if (cbPop)
2433	iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
2434
2435	/* Commit it. */
2436	pVCpu->cpum.GstCtx.rip = NewRip.u;
2437	pVCpu->cpum.GstCtx.rsp = NewRsp.u;
2438
2439	/* Flush the prefetch buffer. */
2440	#ifndef IEM_WITH_CODE_TLB
2441	iemOpcodeFlushLight(pVCpu, cbInstr);
2442	#endif
2443	RT_NOREF(cbInstr);
2444
2445
2446	if (a_fWithFlags)
2447	return iemRegFinishClearingFlags(pVCpu, VINF_SUCCESS);
2448	return iemRegFinishNoFlags(pVCpu, VINF_SUCCESS);
2449	}
2450
2451
2452	/**
2453	* Implements retn and retn imm16.
2454	*
2455	* @param pVCpu The cross context virtual CPU structure of the
2456	* calling thread.
2457	* @param cbInstr The current instruction length.
2458	* @param enmEffOpSize The effective operand size. This is constant.
2459	* @param cbPop The amount of arguments to pop from the stack
2460	* (bytes). This can be constant (zero).
2461	*/
2462	DECL_FORCE_INLINE(VBOXSTRICTRC)
2463	iemRegRipNearReturnAndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT
2464	{
2465	return iemRegRipNearReturnCommon<true /a_fWithFlags/>(pVCpu, cbInstr, cbPop, enmEffOpSize);
2466	}
2467
2468
2469	/**
2470	* Implements retn and retn imm16, no checking or clearing of
2471	* flags.
2472	*
2473	* @param pVCpu The cross context virtual CPU structure of the
2474	* calling thread.
2475	* @param cbInstr The current instruction length.
2476	* @param enmEffOpSize The effective operand size. This is constant.
2477	* @param cbPop The amount of arguments to pop from the stack
2478	* (bytes). This can be constant (zero).
2479	*/
2480	DECL_FORCE_INLINE(VBOXSTRICTRC)
2481	iemRegRipNearReturnAndFinishNoFlags(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t cbPop, IEMMODE enmEffOpSize) RT_NOEXCEPT
2482	{
2483	return iemRegRipNearReturnCommon<false /a_fWithFlags/>(pVCpu, cbInstr, cbPop, enmEffOpSize);
2484	}
2485
2486	#endif /* go over this later */
2487
2488	/** @} */
2489
2490
2491	#if 0 /** @todo go over this later */
2492
2493	/** @name FPU access and helpers.
2494	*
2495	* @{
2496	*/
2497
2498
2499	/**
2500	* Hook for preparing to use the host FPU.
2501	*
2502	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2503	*
2504	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2505	*/
2506	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2507	{
2508	#ifdef IN_RING3
2509	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2510	#else
2511	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2512	#endif
2513	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2514	}
2515
2516
2517	/**
2518	* Hook for preparing to use the host FPU for SSE.
2519	*
2520	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2521	*
2522	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2523	*/
2524	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2525	{
2526	iemFpuPrepareUsage(pVCpu);
2527	}
2528
2529
2530	/**
2531	* Hook for preparing to use the host FPU for AVX.
2532	*
2533	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2534	*
2535	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2536	*/
2537	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2538	{
2539	iemFpuPrepareUsage(pVCpu);
2540	}
2541
2542
2543	/**
2544	* Hook for actualizing the guest FPU state before the interpreter reads it.
2545	*
2546	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2547	*
2548	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2549	*/
2550	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2551	{
2552	#ifdef IN_RING3
2553	NOREF(pVCpu);
2554	#else
2555	CPUMRZFpuStateActualizeForRead(pVCpu);
2556	#endif
2557	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2558	}
2559
2560
2561	/**
2562	* Hook for actualizing the guest FPU state before the interpreter changes it.
2563	*
2564	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2565	*
2566	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2567	*/
2568	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2569	{
2570	#ifdef IN_RING3
2571	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2572	#else
2573	CPUMRZFpuStateActualizeForChange(pVCpu);
2574	#endif
2575	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2576	}
2577
2578
2579	/**
2580	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2581	* only.
2582	*
2583	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2584	*
2585	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2586	*/
2587	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2588	{
2589	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2590	NOREF(pVCpu);
2591	#else
2592	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2593	#endif
2594	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2595	}
2596
2597
2598	/**
2599	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2600	* read+write.
2601	*
2602	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2603	*
2604	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2605	*/
2606	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2607	{
2608	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2609	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2610	#else
2611	CPUMRZFpuStateActualizeForChange(pVCpu);
2612	#endif
2613	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2614
2615	/* Make sure any changes are loaded the next time around. */
2616	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2617	}
2618
2619
2620	/**
2621	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2622	* only.
2623	*
2624	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2625	*
2626	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2627	*/
2628	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2629	{
2630	#ifdef IN_RING3
2631	NOREF(pVCpu);
2632	#else
2633	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2634	#endif
2635	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2636	}
2637
2638
2639	/**
2640	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2641	* read+write.
2642	*
2643	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2644	*
2645	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2646	*/
2647	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2648	{
2649	#ifdef IN_RING3
2650	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2651	#else
2652	CPUMRZFpuStateActualizeForChange(pVCpu);
2653	#endif
2654	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2655
2656	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2657	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2658	}
2659
2660
2661	/**
2662	* Stores a QNaN value into a FPU register.
2663	*
2664	* @param pReg Pointer to the register.
2665	*/
2666	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2667	{
2668	pReg->au32[0] = UINT32_C(0x00000000);
2669	pReg->au32[1] = UINT32_C(0xc0000000);
2670	pReg->au16[4] = UINT16_C(0xffff);
2671	}
2672
2673
2674	/**
2675	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
2676	*
2677	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2678	* @param pFpuCtx The FPU context.
2679	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2680	*/
2681	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2682	{
2683	Assert(uFpuOpcode != UINT16_MAX);
2684	pFpuCtx->FOP = uFpuOpcode;
2685	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2686	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2687	{
2688	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2689	* happens in real mode here based on the fnsave and fnstenv images. */
2690	pFpuCtx->CS = 0;
2691	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2692	}
2693	else if (!IEM_IS_LONG_MODE(pVCpu))
2694	{
2695	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2696	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2697	}
2698	else
2699	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2700	}
2701
2702
2703	/**
2704	* Marks the specified stack register as free (for FFREE).
2705	*
2706	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2707	* @param iStReg The register to free.
2708	*/
2709	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2710	{
2711	Assert(iStReg < 8);
2712	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2713	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2714	pFpuCtx->FTW &= ~RT_BIT(iReg);
2715	}
2716
2717
2718	/**
2719	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2720	*
2721	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2722	*/
2723	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2724	{
2725	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2726	uint16_t uFsw = pFpuCtx->FSW;
2727	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2728	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2729	uFsw &= ~X86_FSW_TOP_MASK;
2730	uFsw \|= uTop;
2731	pFpuCtx->FSW = uFsw;
2732	}
2733
2734
2735	/**
2736	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2737	*
2738	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2739	*/
2740	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2741	{
2742	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2743	uint16_t uFsw = pFpuCtx->FSW;
2744	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2745	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2746	uFsw &= ~X86_FSW_TOP_MASK;
2747	uFsw \|= uTop;
2748	pFpuCtx->FSW = uFsw;
2749	}
2750
2751
2752
2753
2754	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2755	{
2756	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2757	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2758	if (pFpuCtx->FTW & RT_BIT(iReg))
2759	return VINF_SUCCESS;
2760	return VERR_NOT_FOUND;
2761	}
2762
2763
2764	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2765	{
2766	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2767	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2768	if (pFpuCtx->FTW & RT_BIT(iReg))
2769	{
2770	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2771	return VINF_SUCCESS;
2772	}
2773	return VERR_NOT_FOUND;
2774	}
2775
2776
2777	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2778	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2779	{
2780	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2781	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2782	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2783	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2784	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2785	{
2786	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2787	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2788	return VINF_SUCCESS;
2789	}
2790	return VERR_NOT_FOUND;
2791	}
2792
2793
2794	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2795	{
2796	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2797	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2798	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2799	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2800	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2801	{
2802	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2803	return VINF_SUCCESS;
2804	}
2805	return VERR_NOT_FOUND;
2806	}
2807
2808
2809	/**
2810	* Rotates the stack registers when setting new TOS.
2811	*
2812	* @param pFpuCtx The FPU context.
2813	* @param iNewTop New TOS value.
2814	* @remarks We only do this to speed up fxsave/fxrstor which
2815	* arrange the FP registers in stack order.
2816	* MUST be done before writing the new TOS (FSW).
2817	*/
2818	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
2819	{
2820	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2821	RTFLOAT80U ar80Temp[8];
2822
2823	if (iOldTop == iNewTop)
2824	return;
2825
2826	/* Unscrew the stack and get it into 'native' order. */
2827	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
2828	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
2829	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
2830	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
2831	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
2832	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
2833	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
2834	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
2835
2836	/* Now rotate the stack to the new position. */
2837	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
2838	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
2839	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
2840	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
2841	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
2842	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
2843	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
2844	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
2845	}
2846
2847
2848	/**
2849	* Updates the FPU exception status after FCW is changed.
2850	*
2851	* @param pFpuCtx The FPU context.
2852	*/
2853	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
2854	{
2855	uint16_t u16Fsw = pFpuCtx->FSW;
2856	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2857	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2858	else
2859	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2860	pFpuCtx->FSW = u16Fsw;
2861	}
2862
2863
2864	/**
2865	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2866	*
2867	* @returns The full FTW.
2868	* @param pFpuCtx The FPU context.
2869	*/
2870	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
2871	{
2872	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2873	uint16_t u16Ftw = 0;
2874	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2875	for (unsigned iSt = 0; iSt < 8; iSt++)
2876	{
2877	unsigned const iReg = (iSt + iTop) & 7;
2878	if (!(u8Ftw & RT_BIT(iReg)))
2879	u16Ftw \|= 3 << (iReg * 2); /* empty */
2880	else
2881	{
2882	uint16_t uTag;
2883	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2884	if (pr80Reg->s.uExponent == 0x7fff)
2885	uTag = 2; /* Exponent is all 1's => Special. */
2886	else if (pr80Reg->s.uExponent == 0x0000)
2887	{
2888	if (pr80Reg->s.uMantissa == 0x0000)
2889	uTag = 1; /* All bits are zero => Zero. */
2890	else
2891	uTag = 2; /* Must be special. */
2892	}
2893	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2894	uTag = 0; /* Valid. */
2895	else
2896	uTag = 2; /* Must be special. */
2897
2898	u16Ftw \|= uTag << (iReg * 2);
2899	}
2900	}
2901
2902	return u16Ftw;
2903	}
2904
2905
2906	/**
2907	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2908	*
2909	* @returns The compressed FTW.
2910	* @param u16FullFtw The full FTW to convert.
2911	*/
2912	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
2913	{
2914	uint8_t u8Ftw = 0;
2915	for (unsigned i = 0; i < 8; i++)
2916	{
2917	if ((u16FullFtw & 3) != 3 /empty/)
2918	u8Ftw \|= RT_BIT(i);
2919	u16FullFtw >>= 2;
2920	}
2921
2922	return u8Ftw;
2923	}
2924
2925	/** @} */
2926
2927	#endif /* stuff to do later */
2928
2929	#endif /* !VMM_INCLUDED_SRC_VMMAll_target_armv8_IEMInline_armv8_h */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/target-armv8/IEMInline-armv8.h

Download in other formats: