IEMAllN8veRecompBltIn.cpp@ 105231

Last change on this file since 105231 was 104957, checked in by vboxsync, 6 months ago
VMM/IEM: TLB statistics tweaks & fixes. bugref:10687
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 95.6 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 104957 2024-06-18 12:13:41Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	#define IEM_WITH_OPAQUE_DECODER_STATE
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/cpum.h>
38	#include <VBox/vmm/dbgf.h>
39	#include "IEMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/log.h>
42	#include <VBox/err.h>
43	#include <VBox/param.h>
44	#include <iprt/assert.h>
45	#include <iprt/string.h>
46	#if defined(RT_ARCH_AMD64)
47	# include <iprt/x86.h>
48	#elif defined(RT_ARCH_ARM64)
49	# include <iprt/armv8.h>
50	#endif
51
52
53	#include "IEMInline.h"
54	#include "IEMThreadedFunctions.h"
55	#include "IEMN8veRecompiler.h"
56	#include "IEMN8veRecompilerEmit.h"
57	#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61	/*********************************************************************************************************************************
62	* TB Helper Functions *
63	*********************************************************************************************************************************/
64	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
65	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66	#endif
67
68
69	/**
70	* Used by TB code to deal with a TLB miss for a new page.
71	*/
72	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73	{
74	#ifdef IEM_WITH_TLB_STATISTICS
75	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
76	#endif
77	pVCpu->iem.s.pbInstrBuf = NULL;
78	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
79	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
80	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
81	if (pVCpu->iem.s.pbInstrBuf)
82	{ /* likely */ }
83	else
84	{
85	AssertMsgFailed(("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
86	IEM_DO_LONGJMP(pVCpu, VINF_SUCCESS);
87	}
88	}
89
90
91	/**
92	* Used by TB code to deal with a TLB miss for a new page.
93	*/
94	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
95	{
96	#ifdef IEM_WITH_TLB_STATISTICS
97	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
98	#endif
99	pVCpu->iem.s.pbInstrBuf = NULL;
100	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
101	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
102	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
103	AssertMsg(pVCpu->iem.s.pbInstrBuf, ("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
104	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
105	}
106
107
108	/*********************************************************************************************************************************
109	* Builtin functions *
110	*********************************************************************************************************************************/
111
112	/**
113	* Built-in function that does nothing.
114	*
115	* Whether this is called or not can be controlled by the entry in the
116	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
117	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
118	* whether it's the reduced call count in the TBs or the threaded calls flushing
119	* register state.
120	*/
121	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
122	{
123	RT_NOREF(pReNative, pCallEntry);
124	return off;
125	}
126
127	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
128	{
129	pOutgoing = pIncoming;
130	RT_NOREF(pCallEntry);
131	}
132
133
134	/**
135	* Emits for for LogCpuState.
136	*
137	* This shouldn't have any relevant impact on the recompiler state.
138	*/
139	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
140	{
141	#ifdef RT_ARCH_AMD64
142	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
143	/* push rax */
144	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
145	/* push imm32 */
146	pbCodeBuf[off++] = 0x68;
147	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
148	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
149	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
150	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
151	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
152	pbCodeBuf[off++] = X86_OP_REX_W;
153	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
154	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
155	off += sizeof(uint64_t);
156	/* call rax */
157	pbCodeBuf[off++] = 0xff;
158	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
159	/* pop rax */
160	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
161	/* pop rax */
162	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
163	#else
164	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
165	RT_NOREF(pCallEntry);
166	#endif
167
168	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
169	return off;
170	}
171
172	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
173	{
174	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
175	RT_NOREF(pCallEntry);
176	}
177
178
179	/**
180	* Built-in function that calls a C-implemention function taking zero arguments.
181	*/
182	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
183	{
184	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
185	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
186	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
187	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
188	}
189
190	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
191	{
192	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
193	RT_NOREF(pCallEntry);
194	}
195
196
197	/**
198	* Flushes pending writes in preparation of raising an exception or aborting the TB.
199	*/
200	#define BODY_FLUSH_PENDING_WRITES() \
201	off = iemNativeRegFlushPendingWrites(pReNative, off);
202
203
204	/**
205	* Built-in function that checks for pending interrupts that can be delivered or
206	* forced action flags.
207	*
208	* This triggers after the completion of an instruction, so EIP is already at
209	* the next instruction. If an IRQ or important FF is pending, this will return
210	* a non-zero status that stops TB execution.
211	*/
212	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
213	{
214	RT_NOREF(pCallEntry);
215
216	BODY_FLUSH_PENDING_WRITES();
217
218	/* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
219	and I'm too lazy to create a 'Fixed' version of that one. */
220	uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
221	UINT32_MAX, pReNative->uCheckIrqSeqNo++);
222
223	/* Again, we need to load the extended EFLAGS before we actually need them
224	in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
225	loaded them inside the check, as the shadow state would not be correct
226	when the code branches before the load. Ditto PC. */
227	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
228	kIemNativeGstRegUse_ReadOnly);
229
230	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
231
232	uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
233
234	/*
235	* Start by checking the local forced actions of the EMT we're on for IRQs
236	* and other FFs that needs servicing.
237	*/
238	/** @todo this isn't even close to the NMI and interrupt conditions in EM! */
239	/* Load FFs in to idxTmpReg and AND with all relevant flags. */
240	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
241	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
242	VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
243	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
244	\| VMCPU_FF_TLB_FLUSH
245	\| VMCPU_FF_UNHALT ),
246	true /fSetFlags/);
247	/* If we end up with ZERO in idxTmpReg there is nothing to do.*/
248	uint32_t const offFixupJumpToVmCheck1 = off;
249	off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
250
251	/* Some relevant FFs are set, but if's only APIC or/and PIC being set,
252	these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
253	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
254	~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC), true /fSetFlags/);
255	/* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
256	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
257
258	/* So, it's only interrupt releated FFs and we need to see if IRQs are being
259	suppressed by the CPU or not. */
260	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
261	off = iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
262	kIemNativeExitReason_ReturnBreakFF);
263
264	/* We've got shadow flags set, so we must check that the PC they are valid
265	for matches our current PC value. */
266	/** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
267	* a register. */
268	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
269	off = iemNativeEmitTestIfGprNotEqualGprAndTbExit(pReNative, off, idxTmpReg, idxPcReg,
270	kIemNativeExitReason_ReturnBreakFF);
271
272	/*
273	* Now check the force flags of the VM.
274	*/
275	iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
276	iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
277	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
278	off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
279	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /fSetFlags/);
280	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
281
282	/** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
283
284	/*
285	* We're good, no IRQs or FFs pending.
286	*/
287	iemNativeRegFreeTmp(pReNative, idxTmpReg);
288	iemNativeRegFreeTmp(pReNative, idxEflReg);
289	iemNativeRegFreeTmp(pReNative, idxPcReg);
290
291	/*
292	* Note down that we've been here, so we can skip FFs + IRQ checks when
293	* doing direct linking.
294	*/
295	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
296	pReNative->idxLastCheckIrqCallNo = pReNative->idxCurCall;
297	#else
298	pReNative->idxLastCheckIrqCallNo = pCallEntry - pReNative->pTbOrg->Thrd.paCalls;
299	#endif
300
301	return off;
302	}
303
304	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
305	{
306	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
307	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
308	RT_NOREF(pCallEntry);
309	}
310
311
312	/**
313	* Built-in function checks if IEMCPU::fExec has the expected value.
314	*/
315	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
316	{
317	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
318	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
319
320	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
321	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
322	off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
323	kIemNativeExitReason_ReturnBreak);
324	iemNativeRegFreeTmp(pReNative, idxTmpReg);
325
326	/* Maintain the recompiler fExec state. */
327	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
328	return off;
329	}
330
331	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
332	{
333	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
334	RT_NOREF(pCallEntry);
335	}
336
337
338	/**
339	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
340	*/
341	/** @todo Optimize this, so we don't set the same value more than once. Just
342	* needs some tracking. */
343	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
344	# define BODY_SET_CUR_INSTR() \
345	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
346	#else
347	# define BODY_SET_CUR_INSTR() ((void)0)
348	#endif
349
350
351	/**
352	* Macro that emits the 16/32-bit CS.LIM check.
353	*/
354	#define BODY_CHECK_CS_LIM(a_cbInstr) \
355	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
356
357	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
358	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
359
360	DECL_FORCE_INLINE(uint32_t)
361	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
362	{
363	Assert(cbInstr > 0);
364	Assert(cbInstr < 16);
365	#ifdef VBOX_STRICT
366	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
367	#endif
368
369	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
370	Assert(pReNative->Core.offPc == 0);
371	#endif
372
373	/*
374	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
375	* a temporary register for calculating the last address of the instruction.
376	*
377	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
378	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
379	* that last updated EIP here checked it already, and that we're therefore
380	* safe in the 32-bit wrap-around scenario to only check that the last byte
381	* is within CS.LIM. In the case of instruction-by-instruction advancing
382	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
383	* must be using 4KB granularity and the previous instruction was fine.
384	*/
385	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
386	kIemNativeGstRegUse_ReadOnly);
387	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
388	kIemNativeGstRegUse_ReadOnly);
389	#ifdef RT_ARCH_AMD64
390	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
391	#elif defined(RT_ARCH_ARM64)
392	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
393	#else
394	# error "Port me"
395	#endif
396
397	if (cbInstr != 1)
398	{
399	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
400
401	/*
402	* 1. idxRegTmp = idxRegPc + cbInstr;
403	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
404	*/
405	#ifdef RT_ARCH_AMD64
406	/* 1. lea tmp32, [Pc + cbInstr - 1] */
407	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
408	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
409	pbCodeBuf[off++] = 0x8d;
410	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
411	if ((idxRegPc & 7) == X86_GREG_xSP)
412	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
413	pbCodeBuf[off++] = cbInstr - 1;
414
415	/* 2. cmp tmp32(r), CsLim(r/m). */
416	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
417	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
418	pbCodeBuf[off++] = 0x3b;
419	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
420
421	#elif defined(RT_ARCH_ARM64)
422	/* 1. add tmp32, Pc, #cbInstr-1 */
423	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
424	/* 2. cmp tmp32, CsLim */
425	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
426	false /f64Bit/, true /fSetFlags/);
427
428	#endif
429	iemNativeRegFreeTmp(pReNative, idxRegTmp);
430	}
431	else
432	{
433	/*
434	* Here we can skip step 1 and compare PC and CS.LIM directly.
435	*/
436	#ifdef RT_ARCH_AMD64
437	/* 2. cmp eip(r), CsLim(r/m). */
438	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
439	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
440	pbCodeBuf[off++] = 0x3b;
441	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
442
443	#elif defined(RT_ARCH_ARM64)
444	/* 2. cmp Pc, CsLim */
445	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
446	false /f64Bit/, true /fSetFlags/);
447
448	#endif
449	}
450
451	/* 3. Jump if greater. */
452	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
453
454	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
455	iemNativeRegFreeTmp(pReNative, idxRegPc);
456	return off;
457	}
458
459
460	/**
461	* Macro that considers whether we need CS.LIM checking after a branch or
462	* crossing over to a new page.
463	*/
464	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
465	RT_NOREF(a_cbInstr); \
466	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
467
468	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
469	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
470	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
471
472	DECL_FORCE_INLINE(uint32_t)
473	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
474	{
475	#ifdef VBOX_STRICT
476	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
477	#endif
478
479	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
480	Assert(pReNative->Core.offPc == 0);
481	#endif
482
483	/*
484	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
485	* exactly:
486	*
487	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
488	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
489	* return fRet;
490	* return fRet \| IEMTB_F_CS_LIM_CHECKS;
491	*
492	*
493	* We need EIP, CS.LIM and CS.BASE here.
494	*/
495
496	/* Calculate the offFromLim first: */
497	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
498	kIemNativeGstRegUse_ReadOnly);
499	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
500	kIemNativeGstRegUse_ReadOnly);
501	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
502
503	#ifdef RT_ARCH_ARM64
504	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
505	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
506	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
507	#else
508	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
509	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
510	#endif
511
512	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
513	iemNativeRegFreeTmp(pReNative, idxRegPc);
514
515	/* Calculate the threshold level (right side). */
516	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
517	kIemNativeGstRegUse_ReadOnly);
518	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
519
520	#ifdef RT_ARCH_ARM64
521	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
522	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
523	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
524	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
525	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
526	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
527	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
528
529	#else
530	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
531	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
532	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
533	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
534	#endif
535
536	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
537
538	/* Compare the two and jump out if we're too close to the limit. */
539	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
540	off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeExitReason_NeedCsLimChecking);
541
542	iemNativeRegFreeTmp(pReNative, idxRegRight);
543	iemNativeRegFreeTmp(pReNative, idxRegLeft);
544	return off;
545	}
546
547
548
549	/**
550	* Macro that implements opcode (re-)checking.
551	*/
552	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
553	RT_NOREF(a_cbInstr); \
554	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
555
556	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
557
558	#if 0 /* debugging aid */
559	bool g_fBpOnObsoletion = false;
560	# define BP_ON_OBSOLETION g_fBpOnObsoletion
561	#else
562	# define BP_ON_OBSOLETION 0
563	#endif
564
565	DECL_FORCE_INLINE(uint32_t)
566	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
567	{
568	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
569	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
570	#ifdef VBOX_STRICT
571	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
572	#endif
573
574	/*
575	* Where to start and how much to compare.
576	*
577	* Looking at the ranges produced when r160746 was running a DOS VM with TB
578	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
579	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
580	*
581	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
582	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
583	*/
584	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
585	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
586	Assert(cbLeft > 0);
587	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
588	uint32_t offConsolidatedJump = UINT32_MAX;
589
590	#ifdef RT_ARCH_AMD64
591	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
592	completely inlined, for larger we use REPE CMPS. */
593	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
594	pbCodeBuf[off++] = a_bOpcode; \
595	Assert(offPage < 127); \
596	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
597	pbCodeBuf[off++] = RT_BYTE1(offPage); \
598	} while (0)
599
600	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
601	if (offConsolidatedJump != UINT32_MAX) \
602	{ \
603	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
604	Assert(offDisp >= -128); \
605	pbCodeBuf[off++] = 0x75; /* jnz near */ \
606	pbCodeBuf[off++] = (uint8_t)offDisp; \
607	} \
608	else \
609	{ \
610	pbCodeBuf[off++] = 0x74; /* jz near +6 */ \
611	pbCodeBuf[off++] = 0x06 + BP_ON_OBSOLETION; \
612	offConsolidatedJump = off; \
613	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
614	off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeExitReason_ObsoleteTb); \
615	} \
616	} while (0)
617
618	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
619	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
620	pbCodeBuf[off++] = *pbOpcodes++; \
621	pbCodeBuf[off++] = *pbOpcodes++; \
622	pbCodeBuf[off++] = *pbOpcodes++; \
623	pbCodeBuf[off++] = *pbOpcodes++; \
624	cbLeft -= 4; \
625	offPage += 4; \
626	CHECK_OPCODES_CMP_JMP(); \
627	} while (0)
628
629	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
630	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
631	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
632	pbCodeBuf[off++] = *pbOpcodes++; \
633	pbCodeBuf[off++] = *pbOpcodes++; \
634	cbLeft -= 2; \
635	offPage += 2; \
636	CHECK_OPCODES_CMP_JMP(); \
637	} while (0)
638
639	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
640	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
641	pbCodeBuf[off++] = *pbOpcodes++; \
642	cbLeft -= 1; \
643	offPage += 1; \
644	CHECK_OPCODES_CMP_JMP(); \
645	} while (0)
646
647	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
648	if (a_bPrefix) \
649	pbCodeBuf[off++] = (a_bPrefix); \
650	pbCodeBuf[off++] = (a_bOpcode); \
651	CHECK_OPCODES_CMP_JMP(); \
652	cbLeft -= (a_cbToSubtract); \
653	} while (0)
654
655	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
656	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
657	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
658	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
659	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
660	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
661	} while (0)
662
663	if (cbLeft <= 24)
664	{
665	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
666	( RT_BIT_32(X86_GREG_xAX)
667	\| RT_BIT_32(X86_GREG_xCX)
668	\| RT_BIT_32(X86_GREG_xDX)
669	\| RT_BIT_32(X86_GREG_xBX)
670	\| RT_BIT_32(X86_GREG_xSI)
671	\| RT_BIT_32(X86_GREG_xDI))
672	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
673	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
674	if (offPage >= 128 - cbLeft)
675	{
676	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
677	offPage &= 3;
678	}
679
680	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */);
681
682	if (cbLeft > 8)
683	switch (offPage & 3)
684	{
685	case 0:
686	break;
687	case 1: /* cost: 6 + 8 = 14 */
688	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
689	RT_FALL_THRU();
690	case 2: /* cost: 8 */
691	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
692	break;
693	case 3: /* cost: 6 */
694	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
695	break;
696	}
697
698	while (cbLeft >= 4)
699	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
700
701	if (cbLeft >= 2)
702	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
703	if (cbLeft)
704	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
705
706	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
707	iemNativeRegFreeTmp(pReNative, idxRegTmp);
708	}
709	else
710	{
711	/* RDI = &pbInstrBuf[offPage] */
712	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
713	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
714	if (offPage != 0)
715	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
716
717	/* RSI = pbOpcodes */
718	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
719	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
720
721	/* RCX = counts. */
722	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
723
724	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 36/);
725
726	/** @todo profile and optimize this further. Maybe an idea to align by
727	* offPage if the two cannot be reconsidled. */
728	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
729	switch (offPage & 7) /* max cost: 10 */
730	{
731	case 0:
732	break;
733	case 1: /* cost: 3+4+3 = 10 */
734	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
735	RT_FALL_THRU();
736	case 2: /* cost: 4+3 = 7 */
737	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
738	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
739	break;
740	case 3: /* cost: 3+3 = 6 */
741	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
742	RT_FALL_THRU();
743	case 4: /* cost: 3 */
744	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
745	break;
746	case 5: /* cost: 3+4 = 7 */
747	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
748	RT_FALL_THRU();
749	case 6: /* cost: 4 */
750	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
751	break;
752	case 7: /* cost: 3 */
753	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
754	break;
755	}
756
757	/* Compare qwords: */
758	uint32_t const cQWords = cbLeft >> 3;
759	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
760
761	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
762	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
763	cbLeft &= 7;
764
765	if (cbLeft & 4)
766	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
767	if (cbLeft & 2)
768	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
769	if (cbLeft & 1)
770	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
771
772	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
773	iemNativeRegFreeTmp(pReNative, idxRegCx);
774	iemNativeRegFreeTmp(pReNative, idxRegSi);
775	iemNativeRegFreeTmp(pReNative, idxRegDi);
776	}
777
778	#elif defined(RT_ARCH_ARM64)
779	/* We need pbInstrBuf in a register, whatever we do. */
780	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
781	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
782
783	/* We also need at least one more register for holding bytes & words we
784	load via pbInstrBuf. */
785	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
786
787	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
788
789	/* One byte compare can be done with the opcode byte as an immediate. We'll
790	do this to uint16_t align src1. */
791	bool fPendingJmp = RT_BOOL(offPage & 1);
792	if (fPendingJmp)
793	{
794	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
795	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
796	offPage += 1;
797	cbLeft -= 1;
798	}
799
800	if (cbLeft > 0)
801	{
802	/* We need a register for holding the opcode bytes we're comparing with,
803	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
804	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
805
806	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
807	if ((offPage & 3) && cbLeft >= 2)
808	{
809	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
810	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
811	if (fPendingJmp)
812	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
813	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
814	else
815	{
816	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
817	fPendingJmp = true;
818	}
819	pbOpcodes += 2;
820	offPage += 2;
821	cbLeft -= 2;
822	}
823
824	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
825	if ((offPage & 7) && cbLeft >= 4)
826	{
827	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
828	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
829	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
830	if (fPendingJmp)
831	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
832	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
833	else
834	{
835	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
836	fPendingJmp = true;
837	}
838	pbOpcodes += 4;
839	offPage += 4;
840	cbLeft -= 4;
841	}
842
843	/*
844	* If we've got 16 bytes or more left, switch to memcmp-style.
845	*/
846	if (cbLeft >= 16)
847	{
848	/* We need a pointer to the copy of the original opcode bytes. */
849	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
850	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
851
852	/* If there are more than 32 bytes to compare we create a loop, for
853	which we'll need a loop register. */
854	if (cbLeft >= 64)
855	{
856	if (fPendingJmp)
857	{
858	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
859	kArmv8InstrCond_Ne);
860	fPendingJmp = false;
861	}
862
863	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
864	uint16_t const cLoops = cbLeft / 32;
865	cbLeft = cbLeft % 32;
866	pbOpcodes += cLoops * 32;
867	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
868
869	if (offPage != 0) /** @todo optimize out this instruction. */
870	{
871	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
872	offPage = 0;
873	}
874
875	uint32_t const offLoopStart = off;
876	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
877	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
878	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
879
880	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
881	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
882	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
883	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
884
885	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
886	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
887	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
888	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
889
890	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
891	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
892	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
893	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
894
895	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
896	kArmv8InstrCond_Ne);
897
898	/* Advance and loop. */
899	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
900	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
901	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
902	pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
903	off++;
904
905	iemNativeRegFreeTmp(pReNative, idxRegLoop);
906	}
907
908	/* Deal with any remaining dwords (uint64_t). There can be up to
909	three if we looped and four if we didn't. */
910	uint32_t offSrc2 = 0;
911	while (cbLeft >= 8)
912	{
913	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
914	idxRegSrc1Ptr, offPage / 8);
915	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
916	idxRegSrc2Ptr, offSrc2 / 8);
917	if (fPendingJmp)
918	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
919	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
920	else
921	{
922	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
923	fPendingJmp = true;
924	}
925	pbOpcodes += 8;
926	offPage += 8;
927	offSrc2 += 8;
928	cbLeft -= 8;
929	}
930
931	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
932	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
933	}
934	/*
935	* Otherwise, we compare with constants and merge with the general mop-up.
936	*/
937	else
938	{
939	while (cbLeft >= 8)
940	{
941	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
942	offPage / 8);
943	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
944	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
945	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
946	if (fPendingJmp)
947	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
948	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
949	else
950	{
951	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
952	fPendingJmp = true;
953	}
954	pbOpcodes += 8;
955	offPage += 8;
956	cbLeft -= 8;
957	}
958	/* max cost thus far: 21 */
959	}
960
961	/* Deal with any remaining bytes (7 or less). */
962	Assert(cbLeft < 8);
963	if (cbLeft >= 4)
964	{
965	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
966	offPage / 4);
967	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
968	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
969	if (fPendingJmp)
970	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
971	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
972	else
973	{
974	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
975	fPendingJmp = true;
976	}
977	pbOpcodes += 4;
978	offPage += 4;
979	cbLeft -= 4;
980
981	}
982
983	if (cbLeft >= 2)
984	{
985	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
986	offPage / 2);
987	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
988	if (fPendingJmp)
989	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
990	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
991	else
992	{
993	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
994	fPendingJmp = true;
995	}
996	pbOpcodes += 2;
997	offPage += 2;
998	cbLeft -= 2;
999	}
1000
1001	if (cbLeft > 0)
1002	{
1003	Assert(cbLeft == 1);
1004	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
1005	if (fPendingJmp)
1006	{
1007	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
1008	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1009	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1010	}
1011	else
1012	{
1013	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
1014	fPendingJmp = true;
1015	}
1016	pbOpcodes += 1;
1017	offPage += 1;
1018	cbLeft -= 1;
1019	}
1020
1021	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1022	}
1023	Assert(cbLeft == 0);
1024
1025	/*
1026	* Finally, the branch on difference.
1027	*/
1028	if (fPendingJmp)
1029	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ObsoleteTb);
1030
1031	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
1032
1033	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1034	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1035	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1036	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1037
1038	#else
1039	# error "Port me"
1040	#endif
1041	return off;
1042	}
1043
1044
1045
1046	/**
1047	* Macro that implements PC check after a conditional branch.
1048	*/
1049	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1050	RT_NOREF(a_cbInstr); \
1051	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1052
1053	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1054	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1055	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1056	else do { } while (0)
1057
1058	DECL_FORCE_INLINE(uint32_t)
1059	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1060	uint8_t idxRange, uint16_t offRange)
1061	{
1062	#ifdef VBOX_STRICT
1063	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1064	#endif
1065
1066	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1067	Assert(pReNative->Core.offPc == 0);
1068	#endif
1069
1070	/*
1071	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1072	* constant for us here.
1073	*
1074	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1075	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1076	* it, and as long as we don't implement code TLB reload code here there is
1077	* no point in checking that the TLB data we're using is still valid.
1078	*
1079	* What we to do is.
1080	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1081	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1082	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1083	* we're in the wrong spot and need to find a new TB.
1084	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1085	* GCPhysRangePageWithOffset constant mentioned above.
1086	*
1087	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1088	* in 64-bit code or flat 32-bit.
1089	*/
1090
1091	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1092	the temp register, so we don't accidentally clobber something we'll be
1093	needing again immediately. This is why we get idxRegCsBase here. */
1094	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1095	kIemNativeGstRegUse_ReadOnly);
1096	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1097	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1098	kIemNativeGstRegUse_ReadOnly);
1099
1100	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1101
1102	#ifdef VBOX_STRICT
1103	/* Do assertions before idxRegTmp contains anything. */
1104	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1105	# ifdef RT_ARCH_AMD64
1106	{
1107	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1108	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1109	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1110	{
1111	/* cmp r/m64, imm8 */
1112	pbCodeBuf[off++] = X86_OP_REX_W;
1113	pbCodeBuf[off++] = 0x83;
1114	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1115	pbCodeBuf[off++] = 0;
1116	/* je rel8 */
1117	pbCodeBuf[off++] = 0x74;
1118	pbCodeBuf[off++] = 1;
1119	/* int3 */
1120	pbCodeBuf[off++] = 0xcc;
1121
1122	}
1123
1124	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1125	/* test r/m64, imm32 */
1126	pbCodeBuf[off++] = X86_OP_REX_W;
1127	pbCodeBuf[off++] = 0xf7;
1128	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1129	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1130	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1131	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1132	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1133	/* jz rel8 */
1134	pbCodeBuf[off++] = 0x74;
1135	pbCodeBuf[off++] = 1;
1136	/* int3 */
1137	pbCodeBuf[off++] = 0xcc;
1138	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1139	}
1140	# else
1141
1142	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1143	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1144	{
1145	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1146	# ifdef RT_ARCH_ARM64
1147	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1148	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1149	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1150	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1151	# else
1152	# error "Port me!"
1153	# endif
1154	}
1155	# endif
1156
1157	#endif /* VBOX_STRICT */
1158
1159	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1160	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1161	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1162	{
1163	#ifdef RT_ARCH_ARM64
1164	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1165	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1166	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1167	#else
1168	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1169	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1170	#endif
1171	}
1172	else
1173	{
1174	#ifdef RT_ARCH_ARM64
1175	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1176	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1177	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1178	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1179	#else
1180	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1181	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1182	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1183	#endif
1184	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1185	}
1186	iemNativeRegFreeTmp(pReNative, idxRegPc);
1187
1188	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1189	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1190	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
1191
1192	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1193	#ifdef RT_ARCH_AMD64
1194	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1195	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1196	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1197	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1198	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1199
1200	#elif defined(RT_ARCH_ARM64)
1201	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1202
1203	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1204	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1205	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1206	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1207
1208	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1209	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1210	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1211	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1212	# endif
1213	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1214	#else
1215	# error "Port me"
1216	#endif
1217
1218	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1219	\| pTb->aRanges[idxRange].offPhysPage)
1220	+ offRange;
1221	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1222	kIemNativeExitReason_CheckBranchMiss);
1223
1224	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1225	return off;
1226	}
1227
1228
1229	/**
1230	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1231	* instruction crossing into a new page.
1232	*
1233	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1234	*/
1235	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1236	RT_NOREF(a_cbInstr); \
1237	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1238
1239	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1240	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1241	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1242	else do { } while (0)
1243
1244	DECL_FORCE_INLINE(uint32_t)
1245	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1246	{
1247	#ifdef VBOX_STRICT
1248	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1249	#endif
1250
1251	/*
1252	* Define labels and allocate the register for holding the GCPhys of the new page.
1253	*/
1254	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1255	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1256	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1257	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1258	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1259	: UINT32_MAX;
1260
1261	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1262
1263	/*
1264	* Jump to the TLB lookup code.
1265	*/
1266	if (!TlbState.fSkip)
1267	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1268
1269	/*
1270	* TlbMiss:
1271	*
1272	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1273	*/
1274	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1275
1276	/* Save variables in volatile registers. */
1277	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1278	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1279
1280	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1281	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1282
1283	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1284	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1285
1286	/* Done setting up parameters, make the call. */
1287	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1288
1289	/* Move the result to the right register. */
1290	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1291	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1292
1293	/* Restore variables and guest shadow registers to volatile registers. */
1294	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1295	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1296
1297	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1298	if (!TlbState.fSkip)
1299	{
1300	/* end of TlbMiss - Jump to the done label. */
1301	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1302	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1303
1304	/*
1305	* TlbLookup:
1306	*/
1307	off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1308	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1309	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1310	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1311
1312	# ifdef IEM_WITH_TLB_STATISTICS
1313	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1314	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1315	# endif
1316
1317	/*
1318	* TlbDone:
1319	*/
1320	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1321	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1322	}
1323	#else
1324	RT_NOREF(idxLabelTlbMiss);
1325	#endif
1326
1327	/*
1328	* Now check the physical address of the page matches the expected one.
1329	*/
1330	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1331	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1332	kIemNativeExitReason_ObsoleteTb);
1333
1334	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1335	return off;
1336	}
1337
1338
1339	/**
1340	* Macro that implements TLB loading and updating pbInstrBuf updating when
1341	* branching or when crossing a page on an instruction boundrary.
1342	*
1343	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1344	* it is an inter-page branch and also check the page offset.
1345	*
1346	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1347	*/
1348	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1349	RT_NOREF(a_cbInstr); \
1350	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1351
1352	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1353	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1354	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1355	else do { } while (0)
1356
1357	DECL_FORCE_INLINE(uint32_t)
1358	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1359	{
1360	#ifdef VBOX_STRICT
1361	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1362	#endif
1363
1364	BODY_FLUSH_PENDING_WRITES();
1365
1366	/*
1367	* Define labels and allocate the register for holding the GCPhys of the new page.
1368	*/
1369	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1370	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1371	\| pTb->aRanges[idxRange].offPhysPage;
1372
1373	/*
1374	*
1375	* First check if RIP is within the current code.
1376	*
1377	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1378	* difference is what we do when stuff doesn't match up.
1379	*
1380	* What we to do is.
1381	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1382	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1383	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1384	* we need to retranslate RIP via the TLB.
1385	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1386	* GCPhysRangePageWithOffset constant mentioned above.
1387	*
1388	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1389	* in 64-bit code or flat 32-bit.
1390	*
1391	*/
1392
1393	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1394	the temp register, so we don't accidentally clobber something we'll be
1395	needing again immediately. This is why we get idxRegCsBase here.
1396	Update: We share registers with the TlbState, as the TLB code path has
1397	little in common with the rest of the code. */
1398	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1399	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1400	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1401	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1402	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1403	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1404	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1405	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1406
1407	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1408	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1409	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1410
1411	#ifdef VBOX_STRICT
1412	/* Do assertions before idxRegTmp contains anything. */
1413	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1414	# ifdef RT_ARCH_AMD64
1415	{
1416	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1417	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1418	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1419	{
1420	/* cmp r/m64, imm8 */
1421	pbCodeBuf[off++] = X86_OP_REX_W;
1422	pbCodeBuf[off++] = 0x83;
1423	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1424	pbCodeBuf[off++] = 0;
1425	/* je rel8 */
1426	pbCodeBuf[off++] = 0x74;
1427	pbCodeBuf[off++] = 1;
1428	/* int3 */
1429	pbCodeBuf[off++] = 0xcc;
1430
1431	}
1432
1433	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1434	/* test r/m64, imm32 */
1435	pbCodeBuf[off++] = X86_OP_REX_W;
1436	pbCodeBuf[off++] = 0xf7;
1437	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1438	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1439	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1440	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1441	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1442	/* jz rel8 */
1443	pbCodeBuf[off++] = 0x74;
1444	pbCodeBuf[off++] = 1;
1445	/* int3 */
1446	pbCodeBuf[off++] = 0xcc;
1447	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1448	}
1449	# else
1450
1451	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1452	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1453	{
1454	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1455	# ifdef RT_ARCH_ARM64
1456	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1457	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1458	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1459	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1460	# else
1461	# error "Port me!"
1462	# endif
1463	}
1464	# endif
1465
1466	#endif /* VBOX_STRICT */
1467
1468	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1469	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1470	2nd register to check if we've looped more than once already.*/
1471	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1472
1473	uint32_t const offLabelRedoChecks = off;
1474
1475	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1476	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1477	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1478	{
1479	#ifdef RT_ARCH_ARM64
1480	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1481	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1482	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1483	#else
1484	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1485	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1486	#endif
1487	}
1488	else
1489	{
1490	#ifdef RT_ARCH_ARM64
1491	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1492	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1493	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1494	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1495	#else
1496	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1497	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1498	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1499	#endif
1500	}
1501
1502	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1503	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1504	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1505	uint32_t const offFixedJumpToTlbLoad = off;
1506	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1507
1508	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1509	#ifdef RT_ARCH_AMD64
1510	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1511	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1512	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1513	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1514	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1515
1516	#elif defined(RT_ARCH_ARM64)
1517
1518	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1519	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1520	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1521	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1522
1523	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1524	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1525	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1526	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1527	# endif
1528	#else
1529	# error "Port me"
1530	#endif
1531
1532	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1533
1534	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1535	careful and avoid implicit temporary register usage here.
1536
1537	Unlike the threaded version of this code, we do not obsolete TBs here to
1538	reduce the code size and because indirect calls may legally end at the
1539	same offset in two different pages depending on the program state. */
1540	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1541	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1542	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1543	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
1544	uint32_t const offFixedJumpToEnd = off;
1545	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1546
1547	/*
1548	* TlbLoad:
1549	*
1550	* First we try to go via the TLB.
1551	*/
1552	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1553
1554	/* Check that we haven't been here before. */
1555	off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2, false /f64Bit/,
1556	kIemNativeExitReason_CheckBranchMiss);
1557
1558	/* Jump to the TLB lookup code. */
1559	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1560	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1561	: UINT32_MAX;
1562	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1563	if (!TlbState.fSkip)
1564	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1565
1566	/*
1567	* TlbMiss:
1568	*
1569	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1570	*/
1571	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1572	RT_NOREF(idxLabelTlbMiss);
1573
1574	/* Save variables in volatile registers. */
1575	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1576	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1577	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1578
1579	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1580	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1581
1582	/* Done setting up parameters, make the call. */
1583	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1584
1585	/* Restore variables and guest shadow registers to volatile registers. */
1586	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1587	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1588	TlbState.getActiveRegsWithShadows()
1589	\| RT_BIT_32(idxRegPc)
1590	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1591
1592	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1593	if (!TlbState.fSkip)
1594	{
1595	/* end of TlbMiss - Jump to the done label. */
1596	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1597	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1598
1599	/*
1600	* TlbLookup:
1601	*/
1602	off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1603	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1604	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1605
1606	# ifdef IEM_WITH_TLB_STATISTICS
1607	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1608	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1609	# endif
1610
1611	/*
1612	* TlbDone:
1613	*/
1614	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1615	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1616	}
1617	#else
1618	RT_NOREF(idxLabelTlbMiss);
1619	#endif
1620
1621	/* Jmp back to the start and redo the checks. */
1622	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1623	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1624
1625	/*
1626	* End:
1627	*
1628	* The end.
1629	*/
1630	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1631
1632	if (!TlbState.fSkip)
1633	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1634	else
1635	{
1636	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1637	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1638	iemNativeRegFreeTmp(pReNative, idxRegPc);
1639	if (idxRegCsBase != UINT8_MAX)
1640	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1641	}
1642	return off;
1643	}
1644
1645
1646	#ifdef BODY_CHECK_CS_LIM
1647	/**
1648	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1649	* raising a \#GP(0) if this isn't the case.
1650	*/
1651	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1652	{
1653	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1654	BODY_SET_CUR_INSTR();
1655	BODY_FLUSH_PENDING_WRITES();
1656	BODY_CHECK_CS_LIM(cbInstr);
1657	return off;
1658	}
1659
1660	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1661	{
1662	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1663	LIVENESS_CHECK_CS_LIM(pOutgoing);
1664	RT_NOREF(pCallEntry);
1665	}
1666	#endif
1667
1668
1669	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1670	/**
1671	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1672	* that may have modified them.
1673	*/
1674	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1675	{
1676	PCIEMTB const pTb = pReNative->pTbOrg;
1677	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1678	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1679	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1680	BODY_SET_CUR_INSTR();
1681	BODY_FLUSH_PENDING_WRITES();
1682	BODY_CHECK_CS_LIM(cbInstr);
1683	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1684	return off;
1685	}
1686
1687	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1688	{
1689	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1690	LIVENESS_CHECK_CS_LIM(pOutgoing);
1691	LIVENESS_CHECK_OPCODES(pOutgoing);
1692	RT_NOREF(pCallEntry);
1693	}
1694	#endif
1695
1696
1697	#if defined(BODY_CHECK_OPCODES)
1698	/**
1699	* Built-in function for re-checking opcodes after an instruction that may have
1700	* modified them.
1701	*/
1702	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1703	{
1704	PCIEMTB const pTb = pReNative->pTbOrg;
1705	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1706	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1707	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1708	BODY_SET_CUR_INSTR();
1709	BODY_FLUSH_PENDING_WRITES();
1710	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1711	return off;
1712	}
1713
1714	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1715	{
1716	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1717	LIVENESS_CHECK_OPCODES(pOutgoing);
1718	RT_NOREF(pCallEntry);
1719	}
1720	#endif
1721
1722
1723	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1724	/**
1725	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1726	* checking after an instruction that may have modified them.
1727	*/
1728	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1729	{
1730	PCIEMTB const pTb = pReNative->pTbOrg;
1731	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1732	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1733	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1734	BODY_SET_CUR_INSTR();
1735	BODY_FLUSH_PENDING_WRITES();
1736	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1737	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1738	return off;
1739	}
1740
1741	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1742	{
1743	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1744	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1745	LIVENESS_CHECK_OPCODES(pOutgoing);
1746	RT_NOREF(pCallEntry);
1747	}
1748	#endif
1749
1750
1751	/*
1752	* Post-branching checkers.
1753	*/
1754
1755	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1756	/**
1757	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1758	* after conditional branching within the same page.
1759	*
1760	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1761	*/
1762	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1763	{
1764	PCIEMTB const pTb = pReNative->pTbOrg;
1765	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1766	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1767	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1768	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1769	BODY_SET_CUR_INSTR();
1770	BODY_FLUSH_PENDING_WRITES();
1771	BODY_CHECK_CS_LIM(cbInstr);
1772	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1773	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1774	//LogFunc(("okay\n"));
1775	return off;
1776	}
1777
1778	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1779	{
1780	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1781	LIVENESS_CHECK_CS_LIM(pOutgoing);
1782	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1783	LIVENESS_CHECK_OPCODES(pOutgoing);
1784	RT_NOREF(pCallEntry);
1785	}
1786	#endif
1787
1788
1789	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1790	/**
1791	* Built-in function for checking the PC and checking opcodes after conditional
1792	* branching within the same page.
1793	*
1794	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1795	*/
1796	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1797	{
1798	PCIEMTB const pTb = pReNative->pTbOrg;
1799	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1800	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1801	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1802	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1803	BODY_SET_CUR_INSTR();
1804	BODY_FLUSH_PENDING_WRITES();
1805	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1806	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1807	//LogFunc(("okay\n"));
1808	return off;
1809	}
1810
1811	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1812	{
1813	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1814	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1815	LIVENESS_CHECK_OPCODES(pOutgoing);
1816	RT_NOREF(pCallEntry);
1817	}
1818	#endif
1819
1820
1821	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1822	/**
1823	* Built-in function for checking the PC and checking opcodes and considering
1824	* the need for CS.LIM checking after conditional branching within the same
1825	* page.
1826	*
1827	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1828	*/
1829	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1830	{
1831	PCIEMTB const pTb = pReNative->pTbOrg;
1832	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1833	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1834	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1835	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1836	BODY_SET_CUR_INSTR();
1837	BODY_FLUSH_PENDING_WRITES();
1838	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1839	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1840	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1841	//LogFunc(("okay\n"));
1842	return off;
1843	}
1844
1845	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1846	{
1847	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1848	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1849	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1850	LIVENESS_CHECK_OPCODES(pOutgoing);
1851	RT_NOREF(pCallEntry);
1852	}
1853	#endif
1854
1855
1856	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1857	/**
1858	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1859	* transitioning to a different code page.
1860	*
1861	* The code page transition can either be natural over onto the next page (with
1862	* the instruction starting at page offset zero) or by means of branching.
1863	*
1864	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
1865	*/
1866	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1867	{
1868	PCIEMTB const pTb = pReNative->pTbOrg;
1869	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1870	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1871	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1872	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1873	BODY_SET_CUR_INSTR();
1874	BODY_FLUSH_PENDING_WRITES();
1875	BODY_CHECK_CS_LIM(cbInstr);
1876	Assert(offRange == 0);
1877	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1878	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1879	//LogFunc(("okay\n"));
1880	return off;
1881	}
1882
1883	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1884	{
1885	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1886	LIVENESS_CHECK_CS_LIM(pOutgoing);
1887	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1888	LIVENESS_CHECK_OPCODES(pOutgoing);
1889	RT_NOREF(pCallEntry);
1890	}
1891	#endif
1892
1893
1894	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
1895	/**
1896	* Built-in function for loading TLB and checking opcodes when transitioning to
1897	* a different code page.
1898	*
1899	* The code page transition can either be natural over onto the next page (with
1900	* the instruction starting at page offset zero) or by means of branching.
1901	*
1902	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1903	*/
1904	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
1905	{
1906	PCIEMTB const pTb = pReNative->pTbOrg;
1907	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1908	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1909	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1910	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1911	BODY_SET_CUR_INSTR();
1912	BODY_FLUSH_PENDING_WRITES();
1913	Assert(offRange == 0);
1914	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1915	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1916	//LogFunc(("okay\n"));
1917	return off;
1918	}
1919
1920	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
1921	{
1922	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1923	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1924	LIVENESS_CHECK_OPCODES(pOutgoing);
1925	RT_NOREF(pCallEntry);
1926	}
1927	#endif
1928
1929
1930	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1931	/**
1932	* Built-in function for loading TLB and checking opcodes and considering the
1933	* need for CS.LIM checking when transitioning to a different code page.
1934	*
1935	* The code page transition can either be natural over onto the next page (with
1936	* the instruction starting at page offset zero) or by means of branching.
1937	*
1938	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1939	*/
1940	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1941	{
1942	PCIEMTB const pTb = pReNative->pTbOrg;
1943	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1944	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1945	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1946	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1947	BODY_SET_CUR_INSTR();
1948	BODY_FLUSH_PENDING_WRITES();
1949	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1950	Assert(offRange == 0);
1951	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1952	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1953	//LogFunc(("okay\n"));
1954	return off;
1955	}
1956
1957	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1958	{
1959	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1960	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1961	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1962	LIVENESS_CHECK_OPCODES(pOutgoing);
1963	RT_NOREF(pCallEntry);
1964	}
1965	#endif
1966
1967
1968
1969	/*
1970	* Natural page crossing checkers.
1971	*/
1972
1973	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
1974	/**
1975	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
1976	* both pages when transitioning to a different code page.
1977	*
1978	* This is used when the previous instruction requires revalidation of opcodes
1979	* bytes and the current instruction stries a page boundrary with opcode bytes
1980	* in both the old and new page.
1981	*
1982	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
1983	*/
1984	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1985	{
1986	PCIEMTB const pTb = pReNative->pTbOrg;
1987	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1988	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
1989	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
1990	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
1991	uint32_t const idxRange2 = idxRange1 + 1;
1992	BODY_SET_CUR_INSTR();
1993	BODY_FLUSH_PENDING_WRITES();
1994	BODY_CHECK_CS_LIM(cbInstr);
1995	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
1996	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
1997	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
1998	return off;
1999	}
2000
2001	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2002	{
2003	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2004	LIVENESS_CHECK_CS_LIM(pOutgoing);
2005	LIVENESS_CHECK_OPCODES(pOutgoing);
2006	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2007	RT_NOREF(pCallEntry);
2008	}
2009	#endif
2010
2011
2012	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2013	/**
2014	* Built-in function for loading TLB and checking opcodes on both pages when
2015	* transitioning to a different code page.
2016	*
2017	* This is used when the previous instruction requires revalidation of opcodes
2018	* bytes and the current instruction stries a page boundrary with opcode bytes
2019	* in both the old and new page.
2020	*
2021	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2022	*/
2023	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2024	{
2025	PCIEMTB const pTb = pReNative->pTbOrg;
2026	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2027	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2028	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2029	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2030	uint32_t const idxRange2 = idxRange1 + 1;
2031	BODY_SET_CUR_INSTR();
2032	BODY_FLUSH_PENDING_WRITES();
2033	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2034	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2035	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2036	return off;
2037	}
2038
2039	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2040	{
2041	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2042	LIVENESS_CHECK_OPCODES(pOutgoing);
2043	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2044	RT_NOREF(pCallEntry);
2045	}
2046	#endif
2047
2048
2049	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2050	/**
2051	* Built-in function for loading TLB and checking opcodes on both pages and
2052	* considering the need for CS.LIM checking when transitioning to a different
2053	* code page.
2054	*
2055	* This is used when the previous instruction requires revalidation of opcodes
2056	* bytes and the current instruction stries a page boundrary with opcode bytes
2057	* in both the old and new page.
2058	*
2059	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2060	*/
2061	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2062	{
2063	PCIEMTB const pTb = pReNative->pTbOrg;
2064	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2065	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2066	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2067	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2068	uint32_t const idxRange2 = idxRange1 + 1;
2069	BODY_SET_CUR_INSTR();
2070	BODY_FLUSH_PENDING_WRITES();
2071	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2072	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2073	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2074	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2075	return off;
2076	}
2077
2078	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2079	{
2080	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2081	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2082	LIVENESS_CHECK_OPCODES(pOutgoing);
2083	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2084	RT_NOREF(pCallEntry);
2085	}
2086	#endif
2087
2088
2089	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2090	/**
2091	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2092	* advancing naturally to a different code page.
2093	*
2094	* Only opcodes on the new page is checked.
2095	*
2096	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2097	*/
2098	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2099	{
2100	PCIEMTB const pTb = pReNative->pTbOrg;
2101	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2102	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2103	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2104	//uint32_t const offRange1 = (uint32_t)uParam2;
2105	uint32_t const idxRange2 = idxRange1 + 1;
2106	BODY_SET_CUR_INSTR();
2107	BODY_FLUSH_PENDING_WRITES();
2108	BODY_CHECK_CS_LIM(cbInstr);
2109	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2110	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2111	return off;
2112	}
2113
2114	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2115	{
2116	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2117	LIVENESS_CHECK_CS_LIM(pOutgoing);
2118	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2119	LIVENESS_CHECK_OPCODES(pOutgoing);
2120	RT_NOREF(pCallEntry);
2121	}
2122	#endif
2123
2124
2125	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2126	/**
2127	* Built-in function for loading TLB and checking opcodes when advancing
2128	* naturally to a different code page.
2129	*
2130	* Only opcodes on the new page is checked.
2131	*
2132	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2133	*/
2134	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2135	{
2136	PCIEMTB const pTb = pReNative->pTbOrg;
2137	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2138	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2139	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2140	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2141	uint32_t const idxRange2 = idxRange1 + 1;
2142	BODY_SET_CUR_INSTR();
2143	BODY_FLUSH_PENDING_WRITES();
2144	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2145	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2146	return off;
2147	}
2148
2149	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2150	{
2151	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2152	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2153	LIVENESS_CHECK_OPCODES(pOutgoing);
2154	RT_NOREF(pCallEntry);
2155	}
2156	#endif
2157
2158
2159	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2160	/**
2161	* Built-in function for loading TLB and checking opcodes and considering the
2162	* need for CS.LIM checking when advancing naturally to a different code page.
2163	*
2164	* Only opcodes on the new page is checked.
2165	*
2166	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2167	*/
2168	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2169	{
2170	PCIEMTB const pTb = pReNative->pTbOrg;
2171	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2172	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2173	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2174	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2175	uint32_t const idxRange2 = idxRange1 + 1;
2176	BODY_SET_CUR_INSTR();
2177	BODY_FLUSH_PENDING_WRITES();
2178	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2179	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2180	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2181	return off;
2182	}
2183
2184	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2185	{
2186	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2187	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2188	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2189	LIVENESS_CHECK_OPCODES(pOutgoing);
2190	RT_NOREF(pCallEntry);
2191	}
2192	#endif
2193
2194
2195	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2196	/**
2197	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2198	* advancing naturally to a different code page with first instr at byte 0.
2199	*
2200	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2201	*/
2202	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2203	{
2204	PCIEMTB const pTb = pReNative->pTbOrg;
2205	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2206	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2207	BODY_SET_CUR_INSTR();
2208	BODY_FLUSH_PENDING_WRITES();
2209	BODY_CHECK_CS_LIM(cbInstr);
2210	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2211	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2212	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2213	return off;
2214	}
2215
2216	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2217	{
2218	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2219	LIVENESS_CHECK_CS_LIM(pOutgoing);
2220	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2221	LIVENESS_CHECK_OPCODES(pOutgoing);
2222	RT_NOREF(pCallEntry);
2223	}
2224	#endif
2225
2226
2227	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2228	/**
2229	* Built-in function for loading TLB and checking opcodes when advancing
2230	* naturally to a different code page with first instr at byte 0.
2231	*
2232	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2233	*/
2234	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2235	{
2236	PCIEMTB const pTb = pReNative->pTbOrg;
2237	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2238	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2239	BODY_SET_CUR_INSTR();
2240	BODY_FLUSH_PENDING_WRITES();
2241	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2242	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2243	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2244	return off;
2245	}
2246
2247	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2248	{
2249	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2250	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2251	LIVENESS_CHECK_OPCODES(pOutgoing);
2252	RT_NOREF(pCallEntry);
2253	}
2254	#endif
2255
2256
2257	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2258	/**
2259	* Built-in function for loading TLB and checking opcodes and considering the
2260	* need for CS.LIM checking when advancing naturally to a different code page
2261	* with first instr at byte 0.
2262	*
2263	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2264	*/
2265	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2266	{
2267	PCIEMTB const pTb = pReNative->pTbOrg;
2268	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2269	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2270	BODY_SET_CUR_INSTR();
2271	BODY_FLUSH_PENDING_WRITES();
2272	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2273	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2274	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2275	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2276	return off;
2277	}
2278
2279	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2280	{
2281	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2282	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2283	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2284	LIVENESS_CHECK_OPCODES(pOutgoing);
2285	RT_NOREF(pCallEntry);
2286	}
2287	#endif
2288

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp@ 105231

Download in other formats: