IEMAllN8veRecompBltIn.cpp@ 104933

Last change on this file since 104933 was 104877, checked in by vboxsync, 8 months ago
VMM/IEM: Removed double checking of the physical TLB revision in iemOpcodeFetchBytesJmp as it introduces an unnecessary race condition and the fallback for the 2nd check is completely wrong. This was causing the DBGFSTOP in the main IEM loop to hit among other things. Also corrected the incorrect throwing of VINF_IEM_REEXEC_BREAK from iemNativeHlpMemCodeNewPageTlbMiss. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 95.5 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 104877 2024-06-10 15:15:11Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	#define IEM_WITH_OPAQUE_DECODER_STATE
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/cpum.h>
38	#include <VBox/vmm/dbgf.h>
39	#include "IEMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/log.h>
42	#include <VBox/err.h>
43	#include <VBox/param.h>
44	#include <iprt/assert.h>
45	#include <iprt/string.h>
46	#if defined(RT_ARCH_AMD64)
47	# include <iprt/x86.h>
48	#elif defined(RT_ARCH_ARM64)
49	# include <iprt/armv8.h>
50	#endif
51
52
53	#include "IEMInline.h"
54	#include "IEMThreadedFunctions.h"
55	#include "IEMN8veRecompiler.h"
56	#include "IEMN8veRecompilerEmit.h"
57	#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61	/*********************************************************************************************************************************
62	* TB Helper Functions *
63	*********************************************************************************************************************************/
64	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
65	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66	#endif
67
68
69	/**
70	* Used by TB code to deal with a TLB miss for a new page.
71	*/
72	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73	{
74	STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
75	pVCpu->iem.s.pbInstrBuf = NULL;
76	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
77	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
78	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
79	if (pVCpu->iem.s.pbInstrBuf)
80	{ /* likely */ }
81	else
82	{
83	AssertMsgFailed(("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
84	IEM_DO_LONGJMP(pVCpu, VINF_SUCCESS);
85	}
86	}
87
88
89	/**
90	* Used by TB code to deal with a TLB miss for a new page.
91	*/
92	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
93	{
94	STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
95	pVCpu->iem.s.pbInstrBuf = NULL;
96	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
97	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
98	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
99	AssertMsg(pVCpu->iem.s.pbInstrBuf, ("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
100	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
101	}
102
103
104	/*********************************************************************************************************************************
105	* Builtin functions *
106	*********************************************************************************************************************************/
107
108	/**
109	* Built-in function that does nothing.
110	*
111	* Whether this is called or not can be controlled by the entry in the
112	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
113	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
114	* whether it's the reduced call count in the TBs or the threaded calls flushing
115	* register state.
116	*/
117	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
118	{
119	RT_NOREF(pReNative, pCallEntry);
120	return off;
121	}
122
123	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
124	{
125	pOutgoing = pIncoming;
126	RT_NOREF(pCallEntry);
127	}
128
129
130	/**
131	* Emits for for LogCpuState.
132	*
133	* This shouldn't have any relevant impact on the recompiler state.
134	*/
135	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
136	{
137	#ifdef RT_ARCH_AMD64
138	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
139	/* push rax */
140	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
141	/* push imm32 */
142	pbCodeBuf[off++] = 0x68;
143	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
144	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
145	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
146	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
147	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
148	pbCodeBuf[off++] = X86_OP_REX_W;
149	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
150	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
151	off += sizeof(uint64_t);
152	/* call rax */
153	pbCodeBuf[off++] = 0xff;
154	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
155	/* pop rax */
156	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
157	/* pop rax */
158	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
159	#else
160	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
161	RT_NOREF(pCallEntry);
162	#endif
163
164	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
165	return off;
166	}
167
168	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
169	{
170	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
171	RT_NOREF(pCallEntry);
172	}
173
174
175	/**
176	* Built-in function that calls a C-implemention function taking zero arguments.
177	*/
178	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
179	{
180	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
181	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
182	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
183	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
184	}
185
186	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
187	{
188	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
189	RT_NOREF(pCallEntry);
190	}
191
192
193	/**
194	* Flushes pending writes in preparation of raising an exception or aborting the TB.
195	*/
196	#define BODY_FLUSH_PENDING_WRITES() \
197	off = iemNativeRegFlushPendingWrites(pReNative, off);
198
199
200	/**
201	* Built-in function that checks for pending interrupts that can be delivered or
202	* forced action flags.
203	*
204	* This triggers after the completion of an instruction, so EIP is already at
205	* the next instruction. If an IRQ or important FF is pending, this will return
206	* a non-zero status that stops TB execution.
207	*/
208	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
209	{
210	RT_NOREF(pCallEntry);
211
212	BODY_FLUSH_PENDING_WRITES();
213
214	/* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
215	and I'm too lazy to create a 'Fixed' version of that one. */
216	uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
217	UINT32_MAX, pReNative->uCheckIrqSeqNo++);
218
219	/* Again, we need to load the extended EFLAGS before we actually need them
220	in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
221	loaded them inside the check, as the shadow state would not be correct
222	when the code branches before the load. Ditto PC. */
223	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
224	kIemNativeGstRegUse_ReadOnly);
225
226	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
227
228	uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
229
230	/*
231	* Start by checking the local forced actions of the EMT we're on for IRQs
232	* and other FFs that needs servicing.
233	*/
234	/** @todo this isn't even close to the NMI and interrupt conditions in EM! */
235	/* Load FFs in to idxTmpReg and AND with all relevant flags. */
236	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
237	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
238	VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
239	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
240	\| VMCPU_FF_TLB_FLUSH
241	\| VMCPU_FF_UNHALT ),
242	true /fSetFlags/);
243	/* If we end up with ZERO in idxTmpReg there is nothing to do.*/
244	uint32_t const offFixupJumpToVmCheck1 = off;
245	off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
246
247	/* Some relevant FFs are set, but if's only APIC or/and PIC being set,
248	these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
249	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
250	~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC), true /fSetFlags/);
251	/* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
252	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
253
254	/* So, it's only interrupt releated FFs and we need to see if IRQs are being
255	suppressed by the CPU or not. */
256	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
257	off = iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
258	kIemNativeExitReason_ReturnBreakFF);
259
260	/* We've got shadow flags set, so we must check that the PC they are valid
261	for matches our current PC value. */
262	/** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
263	* a register. */
264	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
265	off = iemNativeEmitTestIfGprNotEqualGprAndTbExit(pReNative, off, idxTmpReg, idxPcReg,
266	kIemNativeExitReason_ReturnBreakFF);
267
268	/*
269	* Now check the force flags of the VM.
270	*/
271	iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
272	iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
273	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
274	off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
275	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /fSetFlags/);
276	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ReturnBreakFF);
277
278	/** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
279
280	/*
281	* We're good, no IRQs or FFs pending.
282	*/
283	iemNativeRegFreeTmp(pReNative, idxTmpReg);
284	iemNativeRegFreeTmp(pReNative, idxEflReg);
285	iemNativeRegFreeTmp(pReNative, idxPcReg);
286
287	/*
288	* Note down that we've been here, so we can skip FFs + IRQ checks when
289	* doing direct linking.
290	*/
291	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
292	pReNative->idxLastCheckIrqCallNo = pReNative->idxCurCall;
293	#else
294	pReNative->idxLastCheckIrqCallNo = pCallEntry - pReNative->pTbOrg->Thrd.paCalls;
295	#endif
296
297	return off;
298	}
299
300	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
301	{
302	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
303	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
304	RT_NOREF(pCallEntry);
305	}
306
307
308	/**
309	* Built-in function checks if IEMCPU::fExec has the expected value.
310	*/
311	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
312	{
313	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
314	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
315
316	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
317	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
318	off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
319	kIemNativeExitReason_ReturnBreak);
320	iemNativeRegFreeTmp(pReNative, idxTmpReg);
321
322	/* Maintain the recompiler fExec state. */
323	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
324	return off;
325	}
326
327	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
328	{
329	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
330	RT_NOREF(pCallEntry);
331	}
332
333
334	/**
335	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
336	*/
337	/** @todo Optimize this, so we don't set the same value more than once. Just
338	* needs some tracking. */
339	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
340	# define BODY_SET_CUR_INSTR() \
341	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
342	#else
343	# define BODY_SET_CUR_INSTR() ((void)0)
344	#endif
345
346
347	/**
348	* Macro that emits the 16/32-bit CS.LIM check.
349	*/
350	#define BODY_CHECK_CS_LIM(a_cbInstr) \
351	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
352
353	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
354	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
355
356	DECL_FORCE_INLINE(uint32_t)
357	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
358	{
359	Assert(cbInstr > 0);
360	Assert(cbInstr < 16);
361	#ifdef VBOX_STRICT
362	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
363	#endif
364
365	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
366	Assert(pReNative->Core.offPc == 0);
367	#endif
368
369	/*
370	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
371	* a temporary register for calculating the last address of the instruction.
372	*
373	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
374	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
375	* that last updated EIP here checked it already, and that we're therefore
376	* safe in the 32-bit wrap-around scenario to only check that the last byte
377	* is within CS.LIM. In the case of instruction-by-instruction advancing
378	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
379	* must be using 4KB granularity and the previous instruction was fine.
380	*/
381	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
382	kIemNativeGstRegUse_ReadOnly);
383	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
384	kIemNativeGstRegUse_ReadOnly);
385	#ifdef RT_ARCH_AMD64
386	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
387	#elif defined(RT_ARCH_ARM64)
388	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
389	#else
390	# error "Port me"
391	#endif
392
393	if (cbInstr != 1)
394	{
395	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
396
397	/*
398	* 1. idxRegTmp = idxRegPc + cbInstr;
399	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
400	*/
401	#ifdef RT_ARCH_AMD64
402	/* 1. lea tmp32, [Pc + cbInstr - 1] */
403	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
404	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
405	pbCodeBuf[off++] = 0x8d;
406	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
407	if ((idxRegPc & 7) == X86_GREG_xSP)
408	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
409	pbCodeBuf[off++] = cbInstr - 1;
410
411	/* 2. cmp tmp32(r), CsLim(r/m). */
412	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
413	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
414	pbCodeBuf[off++] = 0x3b;
415	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
416
417	#elif defined(RT_ARCH_ARM64)
418	/* 1. add tmp32, Pc, #cbInstr-1 */
419	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
420	/* 2. cmp tmp32, CsLim */
421	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
422	false /f64Bit/, true /fSetFlags/);
423
424	#endif
425	iemNativeRegFreeTmp(pReNative, idxRegTmp);
426	}
427	else
428	{
429	/*
430	* Here we can skip step 1 and compare PC and CS.LIM directly.
431	*/
432	#ifdef RT_ARCH_AMD64
433	/* 2. cmp eip(r), CsLim(r/m). */
434	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
435	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
436	pbCodeBuf[off++] = 0x3b;
437	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
438
439	#elif defined(RT_ARCH_ARM64)
440	/* 2. cmp Pc, CsLim */
441	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
442	false /f64Bit/, true /fSetFlags/);
443
444	#endif
445	}
446
447	/* 3. Jump if greater. */
448	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_RaiseGp0);
449
450	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
451	iemNativeRegFreeTmp(pReNative, idxRegPc);
452	return off;
453	}
454
455
456	/**
457	* Macro that considers whether we need CS.LIM checking after a branch or
458	* crossing over to a new page.
459	*/
460	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
461	RT_NOREF(a_cbInstr); \
462	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
463
464	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
465	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
466	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
467
468	DECL_FORCE_INLINE(uint32_t)
469	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
470	{
471	#ifdef VBOX_STRICT
472	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
473	#endif
474
475	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
476	Assert(pReNative->Core.offPc == 0);
477	#endif
478
479	/*
480	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
481	* exactly:
482	*
483	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
484	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
485	* return fRet;
486	* return fRet \| IEMTB_F_CS_LIM_CHECKS;
487	*
488	*
489	* We need EIP, CS.LIM and CS.BASE here.
490	*/
491
492	/* Calculate the offFromLim first: */
493	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
494	kIemNativeGstRegUse_ReadOnly);
495	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
496	kIemNativeGstRegUse_ReadOnly);
497	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
498
499	#ifdef RT_ARCH_ARM64
500	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
501	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
502	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
503	#else
504	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
505	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
506	#endif
507
508	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
509	iemNativeRegFreeTmp(pReNative, idxRegPc);
510
511	/* Calculate the threshold level (right side). */
512	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
513	kIemNativeGstRegUse_ReadOnly);
514	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
515
516	#ifdef RT_ARCH_ARM64
517	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
518	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
519	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
520	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
521	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
522	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
523	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
524
525	#else
526	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
527	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
528	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
529	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
530	#endif
531
532	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
533
534	/* Compare the two and jump out if we're too close to the limit. */
535	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
536	off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeExitReason_NeedCsLimChecking);
537
538	iemNativeRegFreeTmp(pReNative, idxRegRight);
539	iemNativeRegFreeTmp(pReNative, idxRegLeft);
540	return off;
541	}
542
543
544
545	/**
546	* Macro that implements opcode (re-)checking.
547	*/
548	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
549	RT_NOREF(a_cbInstr); \
550	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
551
552	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
553
554	#if 0 /* debugging aid */
555	bool g_fBpOnObsoletion = false;
556	# define BP_ON_OBSOLETION g_fBpOnObsoletion
557	#else
558	# define BP_ON_OBSOLETION 0
559	#endif
560
561	DECL_FORCE_INLINE(uint32_t)
562	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
563	{
564	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
565	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
566	#ifdef VBOX_STRICT
567	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
568	#endif
569
570	/*
571	* Where to start and how much to compare.
572	*
573	* Looking at the ranges produced when r160746 was running a DOS VM with TB
574	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
575	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
576	*
577	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
578	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
579	*/
580	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
581	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
582	Assert(cbLeft > 0);
583	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
584	uint32_t offConsolidatedJump = UINT32_MAX;
585
586	#ifdef RT_ARCH_AMD64
587	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
588	completely inlined, for larger we use REPE CMPS. */
589	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
590	pbCodeBuf[off++] = a_bOpcode; \
591	Assert(offPage < 127); \
592	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
593	pbCodeBuf[off++] = RT_BYTE1(offPage); \
594	} while (0)
595
596	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
597	if (offConsolidatedJump != UINT32_MAX) \
598	{ \
599	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
600	Assert(offDisp >= -128); \
601	pbCodeBuf[off++] = 0x75; /* jnz near */ \
602	pbCodeBuf[off++] = (uint8_t)offDisp; \
603	} \
604	else \
605	{ \
606	pbCodeBuf[off++] = 0x74; /* jz near +6 */ \
607	pbCodeBuf[off++] = 0x06 + BP_ON_OBSOLETION; \
608	offConsolidatedJump = off; \
609	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
610	off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeExitReason_ObsoleteTb); \
611	} \
612	} while (0)
613
614	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
615	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
616	pbCodeBuf[off++] = *pbOpcodes++; \
617	pbCodeBuf[off++] = *pbOpcodes++; \
618	pbCodeBuf[off++] = *pbOpcodes++; \
619	pbCodeBuf[off++] = *pbOpcodes++; \
620	cbLeft -= 4; \
621	offPage += 4; \
622	CHECK_OPCODES_CMP_JMP(); \
623	} while (0)
624
625	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
626	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
627	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
628	pbCodeBuf[off++] = *pbOpcodes++; \
629	pbCodeBuf[off++] = *pbOpcodes++; \
630	cbLeft -= 2; \
631	offPage += 2; \
632	CHECK_OPCODES_CMP_JMP(); \
633	} while (0)
634
635	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
636	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
637	pbCodeBuf[off++] = *pbOpcodes++; \
638	cbLeft -= 1; \
639	offPage += 1; \
640	CHECK_OPCODES_CMP_JMP(); \
641	} while (0)
642
643	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
644	if (a_bPrefix) \
645	pbCodeBuf[off++] = (a_bPrefix); \
646	pbCodeBuf[off++] = (a_bOpcode); \
647	CHECK_OPCODES_CMP_JMP(); \
648	cbLeft -= (a_cbToSubtract); \
649	} while (0)
650
651	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
652	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
653	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
654	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
655	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
656	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
657	} while (0)
658
659	if (cbLeft <= 24)
660	{
661	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
662	( RT_BIT_32(X86_GREG_xAX)
663	\| RT_BIT_32(X86_GREG_xCX)
664	\| RT_BIT_32(X86_GREG_xDX)
665	\| RT_BIT_32(X86_GREG_xBX)
666	\| RT_BIT_32(X86_GREG_xSI)
667	\| RT_BIT_32(X86_GREG_xDI))
668	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
669	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
670	if (offPage >= 128 - cbLeft)
671	{
672	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
673	offPage &= 3;
674	}
675
676	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */);
677
678	if (cbLeft > 8)
679	switch (offPage & 3)
680	{
681	case 0:
682	break;
683	case 1: /* cost: 6 + 8 = 14 */
684	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
685	RT_FALL_THRU();
686	case 2: /* cost: 8 */
687	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
688	break;
689	case 3: /* cost: 6 */
690	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
691	break;
692	}
693
694	while (cbLeft >= 4)
695	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
696
697	if (cbLeft >= 2)
698	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
699	if (cbLeft)
700	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
701
702	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
703	iemNativeRegFreeTmp(pReNative, idxRegTmp);
704	}
705	else
706	{
707	/* RDI = &pbInstrBuf[offPage] */
708	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
709	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
710	if (offPage != 0)
711	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
712
713	/* RSI = pbOpcodes */
714	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
715	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
716
717	/* RCX = counts. */
718	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
719
720	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 36/);
721
722	/** @todo profile and optimize this further. Maybe an idea to align by
723	* offPage if the two cannot be reconsidled. */
724	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
725	switch (offPage & 7) /* max cost: 10 */
726	{
727	case 0:
728	break;
729	case 1: /* cost: 3+4+3 = 10 */
730	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
731	RT_FALL_THRU();
732	case 2: /* cost: 4+3 = 7 */
733	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
734	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
735	break;
736	case 3: /* cost: 3+3 = 6 */
737	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
738	RT_FALL_THRU();
739	case 4: /* cost: 3 */
740	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
741	break;
742	case 5: /* cost: 3+4 = 7 */
743	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
744	RT_FALL_THRU();
745	case 6: /* cost: 4 */
746	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
747	break;
748	case 7: /* cost: 3 */
749	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
750	break;
751	}
752
753	/* Compare qwords: */
754	uint32_t const cQWords = cbLeft >> 3;
755	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
756
757	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
758	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
759	cbLeft &= 7;
760
761	if (cbLeft & 4)
762	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
763	if (cbLeft & 2)
764	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
765	if (cbLeft & 1)
766	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
767
768	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
769	iemNativeRegFreeTmp(pReNative, idxRegCx);
770	iemNativeRegFreeTmp(pReNative, idxRegSi);
771	iemNativeRegFreeTmp(pReNative, idxRegDi);
772	}
773
774	#elif defined(RT_ARCH_ARM64)
775	/* We need pbInstrBuf in a register, whatever we do. */
776	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
777	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
778
779	/* We also need at least one more register for holding bytes & words we
780	load via pbInstrBuf. */
781	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
782
783	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
784
785	/* One byte compare can be done with the opcode byte as an immediate. We'll
786	do this to uint16_t align src1. */
787	bool fPendingJmp = RT_BOOL(offPage & 1);
788	if (fPendingJmp)
789	{
790	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
791	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
792	offPage += 1;
793	cbLeft -= 1;
794	}
795
796	if (cbLeft > 0)
797	{
798	/* We need a register for holding the opcode bytes we're comparing with,
799	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
800	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
801
802	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
803	if ((offPage & 3) && cbLeft >= 2)
804	{
805	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
806	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
807	if (fPendingJmp)
808	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
809	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
810	else
811	{
812	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
813	fPendingJmp = true;
814	}
815	pbOpcodes += 2;
816	offPage += 2;
817	cbLeft -= 2;
818	}
819
820	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
821	if ((offPage & 7) && cbLeft >= 4)
822	{
823	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
824	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
825	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
826	if (fPendingJmp)
827	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
828	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
829	else
830	{
831	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
832	fPendingJmp = true;
833	}
834	pbOpcodes += 4;
835	offPage += 4;
836	cbLeft -= 4;
837	}
838
839	/*
840	* If we've got 16 bytes or more left, switch to memcmp-style.
841	*/
842	if (cbLeft >= 16)
843	{
844	/* We need a pointer to the copy of the original opcode bytes. */
845	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
846	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
847
848	/* If there are more than 32 bytes to compare we create a loop, for
849	which we'll need a loop register. */
850	if (cbLeft >= 64)
851	{
852	if (fPendingJmp)
853	{
854	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
855	kArmv8InstrCond_Ne);
856	fPendingJmp = false;
857	}
858
859	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
860	uint16_t const cLoops = cbLeft / 32;
861	cbLeft = cbLeft % 32;
862	pbOpcodes += cLoops * 32;
863	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
864
865	if (offPage != 0) /** @todo optimize out this instruction. */
866	{
867	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
868	offPage = 0;
869	}
870
871	uint32_t const offLoopStart = off;
872	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
873	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
874	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
875
876	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
877	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
878	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
879	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
880
881	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
882	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
883	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
884	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
885
886	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
887	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
888	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
889	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
890
891	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeExitReason_ObsoleteTb,
892	kArmv8InstrCond_Ne);
893
894	/* Advance and loop. */
895	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
896	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
897	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
898	pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
899	off++;
900
901	iemNativeRegFreeTmp(pReNative, idxRegLoop);
902	}
903
904	/* Deal with any remaining dwords (uint64_t). There can be up to
905	three if we looped and four if we didn't. */
906	uint32_t offSrc2 = 0;
907	while (cbLeft >= 8)
908	{
909	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
910	idxRegSrc1Ptr, offPage / 8);
911	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
912	idxRegSrc2Ptr, offSrc2 / 8);
913	if (fPendingJmp)
914	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
915	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
916	else
917	{
918	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
919	fPendingJmp = true;
920	}
921	pbOpcodes += 8;
922	offPage += 8;
923	offSrc2 += 8;
924	cbLeft -= 8;
925	}
926
927	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
928	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
929	}
930	/*
931	* Otherwise, we compare with constants and merge with the general mop-up.
932	*/
933	else
934	{
935	while (cbLeft >= 8)
936	{
937	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
938	offPage / 8);
939	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
940	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
941	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
942	if (fPendingJmp)
943	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
944	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
945	else
946	{
947	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
948	fPendingJmp = true;
949	}
950	pbOpcodes += 8;
951	offPage += 8;
952	cbLeft -= 8;
953	}
954	/* max cost thus far: 21 */
955	}
956
957	/* Deal with any remaining bytes (7 or less). */
958	Assert(cbLeft < 8);
959	if (cbLeft >= 4)
960	{
961	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
962	offPage / 4);
963	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
964	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
965	if (fPendingJmp)
966	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
967	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
968	else
969	{
970	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
971	fPendingJmp = true;
972	}
973	pbOpcodes += 4;
974	offPage += 4;
975	cbLeft -= 4;
976
977	}
978
979	if (cbLeft >= 2)
980	{
981	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
982	offPage / 2);
983	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
984	if (fPendingJmp)
985	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
986	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
987	else
988	{
989	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
990	fPendingJmp = true;
991	}
992	pbOpcodes += 2;
993	offPage += 2;
994	cbLeft -= 2;
995	}
996
997	if (cbLeft > 0)
998	{
999	Assert(cbLeft == 1);
1000	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
1001	if (fPendingJmp)
1002	{
1003	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
1004	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1005	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1006	}
1007	else
1008	{
1009	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
1010	fPendingJmp = true;
1011	}
1012	pbOpcodes += 1;
1013	offPage += 1;
1014	cbLeft -= 1;
1015	}
1016
1017	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1018	}
1019	Assert(cbLeft == 0);
1020
1021	/*
1022	* Finally, the branch on difference.
1023	*/
1024	if (fPendingJmp)
1025	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_ObsoleteTb);
1026
1027	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
1028
1029	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1030	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1031	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1032	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1033
1034	#else
1035	# error "Port me"
1036	#endif
1037	return off;
1038	}
1039
1040
1041
1042	/**
1043	* Macro that implements PC check after a conditional branch.
1044	*/
1045	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1046	RT_NOREF(a_cbInstr); \
1047	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1048
1049	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1050	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1051	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1052	else do { } while (0)
1053
1054	DECL_FORCE_INLINE(uint32_t)
1055	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1056	uint8_t idxRange, uint16_t offRange)
1057	{
1058	#ifdef VBOX_STRICT
1059	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1060	#endif
1061
1062	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1063	Assert(pReNative->Core.offPc == 0);
1064	#endif
1065
1066	/*
1067	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1068	* constant for us here.
1069	*
1070	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1071	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1072	* it, and as long as we don't implement code TLB reload code here there is
1073	* no point in checking that the TLB data we're using is still valid.
1074	*
1075	* What we to do is.
1076	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1077	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1078	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1079	* we're in the wrong spot and need to find a new TB.
1080	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1081	* GCPhysRangePageWithOffset constant mentioned above.
1082	*
1083	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1084	* in 64-bit code or flat 32-bit.
1085	*/
1086
1087	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1088	the temp register, so we don't accidentally clobber something we'll be
1089	needing again immediately. This is why we get idxRegCsBase here. */
1090	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1091	kIemNativeGstRegUse_ReadOnly);
1092	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1093	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1094	kIemNativeGstRegUse_ReadOnly);
1095
1096	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1097
1098	#ifdef VBOX_STRICT
1099	/* Do assertions before idxRegTmp contains anything. */
1100	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1101	# ifdef RT_ARCH_AMD64
1102	{
1103	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1104	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1105	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1106	{
1107	/* cmp r/m64, imm8 */
1108	pbCodeBuf[off++] = X86_OP_REX_W;
1109	pbCodeBuf[off++] = 0x83;
1110	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1111	pbCodeBuf[off++] = 0;
1112	/* je rel8 */
1113	pbCodeBuf[off++] = 0x74;
1114	pbCodeBuf[off++] = 1;
1115	/* int3 */
1116	pbCodeBuf[off++] = 0xcc;
1117
1118	}
1119
1120	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1121	/* test r/m64, imm32 */
1122	pbCodeBuf[off++] = X86_OP_REX_W;
1123	pbCodeBuf[off++] = 0xf7;
1124	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1125	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1126	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1127	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1128	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1129	/* jz rel8 */
1130	pbCodeBuf[off++] = 0x74;
1131	pbCodeBuf[off++] = 1;
1132	/* int3 */
1133	pbCodeBuf[off++] = 0xcc;
1134	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1135	}
1136	# else
1137
1138	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1139	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1140	{
1141	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1142	# ifdef RT_ARCH_ARM64
1143	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1144	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1145	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1146	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1147	# else
1148	# error "Port me!"
1149	# endif
1150	}
1151	# endif
1152
1153	#endif /* VBOX_STRICT */
1154
1155	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1156	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1157	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1158	{
1159	#ifdef RT_ARCH_ARM64
1160	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1161	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1162	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1163	#else
1164	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1165	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1166	#endif
1167	}
1168	else
1169	{
1170	#ifdef RT_ARCH_ARM64
1171	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1172	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1173	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1174	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1175	#else
1176	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1177	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1178	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1179	#endif
1180	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1181	}
1182	iemNativeRegFreeTmp(pReNative, idxRegPc);
1183
1184	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1185	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1186	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
1187
1188	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1189	#ifdef RT_ARCH_AMD64
1190	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1191	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1192	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1193	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1194	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1195
1196	#elif defined(RT_ARCH_ARM64)
1197	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1198
1199	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1200	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1201	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1202	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1203
1204	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1205	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1206	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1207	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1208	# endif
1209	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1210	#else
1211	# error "Port me"
1212	#endif
1213
1214	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1215	\| pTb->aRanges[idxRange].offPhysPage)
1216	+ offRange;
1217	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1218	kIemNativeExitReason_CheckBranchMiss);
1219
1220	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1221	return off;
1222	}
1223
1224
1225	/**
1226	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1227	* instruction crossing into a new page.
1228	*
1229	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1230	*/
1231	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1232	RT_NOREF(a_cbInstr); \
1233	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1234
1235	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1236	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1237	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1238	else do { } while (0)
1239
1240	DECL_FORCE_INLINE(uint32_t)
1241	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1242	{
1243	#ifdef VBOX_STRICT
1244	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1245	#endif
1246
1247	/*
1248	* Define labels and allocate the register for holding the GCPhys of the new page.
1249	*/
1250	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1251	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1252	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1253	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1254	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1255	: UINT32_MAX;
1256
1257	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1258
1259	/*
1260	* Jump to the TLB lookup code.
1261	*/
1262	if (!TlbState.fSkip)
1263	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1264
1265	/*
1266	* TlbMiss:
1267	*
1268	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1269	*/
1270	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1271
1272	/* Save variables in volatile registers. */
1273	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1274	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1275
1276	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1277	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1278
1279	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1280	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1281
1282	/* Done setting up parameters, make the call. */
1283	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1284
1285	/* Move the result to the right register. */
1286	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1287	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1288
1289	/* Restore variables and guest shadow registers to volatile registers. */
1290	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1291	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1292
1293	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1294	if (!TlbState.fSkip)
1295	{
1296	/* end of TlbMiss - Jump to the done label. */
1297	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1298	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1299
1300	/*
1301	* TlbLookup:
1302	*/
1303	off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1304	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1305	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1306	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1307
1308	# ifdef VBOX_WITH_STATISTICS
1309	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1310	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1311	# endif
1312
1313	/*
1314	* TlbDone:
1315	*/
1316	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1317	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1318	}
1319	#else
1320	RT_NOREF(idxLabelTlbMiss);
1321	#endif
1322
1323	/*
1324	* Now check the physical address of the page matches the expected one.
1325	*/
1326	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1327	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1328	kIemNativeExitReason_ObsoleteTb);
1329
1330	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1331	return off;
1332	}
1333
1334
1335	/**
1336	* Macro that implements TLB loading and updating pbInstrBuf updating when
1337	* branching or when crossing a page on an instruction boundrary.
1338	*
1339	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1340	* it is an inter-page branch and also check the page offset.
1341	*
1342	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1343	*/
1344	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1345	RT_NOREF(a_cbInstr); \
1346	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1347
1348	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1349	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1350	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1351	else do { } while (0)
1352
1353	DECL_FORCE_INLINE(uint32_t)
1354	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1355	{
1356	#ifdef VBOX_STRICT
1357	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1358	#endif
1359
1360	BODY_FLUSH_PENDING_WRITES();
1361
1362	/*
1363	* Define labels and allocate the register for holding the GCPhys of the new page.
1364	*/
1365	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1366	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1367	\| pTb->aRanges[idxRange].offPhysPage;
1368
1369	/*
1370	*
1371	* First check if RIP is within the current code.
1372	*
1373	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1374	* difference is what we do when stuff doesn't match up.
1375	*
1376	* What we to do is.
1377	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1378	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1379	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1380	* we need to retranslate RIP via the TLB.
1381	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1382	* GCPhysRangePageWithOffset constant mentioned above.
1383	*
1384	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1385	* in 64-bit code or flat 32-bit.
1386	*
1387	*/
1388
1389	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1390	the temp register, so we don't accidentally clobber something we'll be
1391	needing again immediately. This is why we get idxRegCsBase here.
1392	Update: We share registers with the TlbState, as the TLB code path has
1393	little in common with the rest of the code. */
1394	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1395	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1396	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1397	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1398	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1399	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1400	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1401	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1402
1403	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1404	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1405	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1406
1407	#ifdef VBOX_STRICT
1408	/* Do assertions before idxRegTmp contains anything. */
1409	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1410	# ifdef RT_ARCH_AMD64
1411	{
1412	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1413	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1414	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1415	{
1416	/* cmp r/m64, imm8 */
1417	pbCodeBuf[off++] = X86_OP_REX_W;
1418	pbCodeBuf[off++] = 0x83;
1419	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1420	pbCodeBuf[off++] = 0;
1421	/* je rel8 */
1422	pbCodeBuf[off++] = 0x74;
1423	pbCodeBuf[off++] = 1;
1424	/* int3 */
1425	pbCodeBuf[off++] = 0xcc;
1426
1427	}
1428
1429	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1430	/* test r/m64, imm32 */
1431	pbCodeBuf[off++] = X86_OP_REX_W;
1432	pbCodeBuf[off++] = 0xf7;
1433	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1434	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1435	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1436	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1437	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1438	/* jz rel8 */
1439	pbCodeBuf[off++] = 0x74;
1440	pbCodeBuf[off++] = 1;
1441	/* int3 */
1442	pbCodeBuf[off++] = 0xcc;
1443	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1444	}
1445	# else
1446
1447	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1448	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1449	{
1450	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1451	# ifdef RT_ARCH_ARM64
1452	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1453	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1454	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1455	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1456	# else
1457	# error "Port me!"
1458	# endif
1459	}
1460	# endif
1461
1462	#endif /* VBOX_STRICT */
1463
1464	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1465	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1466	2nd register to check if we've looped more than once already.*/
1467	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1468
1469	uint32_t const offLabelRedoChecks = off;
1470
1471	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1472	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1473	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1474	{
1475	#ifdef RT_ARCH_ARM64
1476	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1477	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1478	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1479	#else
1480	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1481	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1482	#endif
1483	}
1484	else
1485	{
1486	#ifdef RT_ARCH_ARM64
1487	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1488	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1489	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1490	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1491	#else
1492	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1493	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1494	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1495	#endif
1496	}
1497
1498	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1499	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1500	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1501	uint32_t const offFixedJumpToTlbLoad = off;
1502	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1503
1504	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1505	#ifdef RT_ARCH_AMD64
1506	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1507	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1508	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1509	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1510	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1511
1512	#elif defined(RT_ARCH_ARM64)
1513
1514	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1515	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1516	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1517	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1518
1519	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1520	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1521	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1522	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1523	# endif
1524	#else
1525	# error "Port me"
1526	#endif
1527
1528	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1529
1530	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1531	careful and avoid implicit temporary register usage here.
1532
1533	Unlike the threaded version of this code, we do not obsolete TBs here to
1534	reduce the code size and because indirect calls may legally end at the
1535	same offset in two different pages depending on the program state. */
1536	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1537	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1538	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1539	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeExitReason_CheckBranchMiss);
1540	uint32_t const offFixedJumpToEnd = off;
1541	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1542
1543	/*
1544	* TlbLoad:
1545	*
1546	* First we try to go via the TLB.
1547	*/
1548	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1549
1550	/* Check that we haven't been here before. */
1551	off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2, false /f64Bit/,
1552	kIemNativeExitReason_CheckBranchMiss);
1553
1554	/* Jump to the TLB lookup code. */
1555	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1556	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1557	: UINT32_MAX;
1558	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1559	if (!TlbState.fSkip)
1560	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1561
1562	/*
1563	* TlbMiss:
1564	*
1565	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1566	*/
1567	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1568	RT_NOREF(idxLabelTlbMiss);
1569
1570	/* Save variables in volatile registers. */
1571	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1572	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1573	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1574
1575	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1576	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1577
1578	/* Done setting up parameters, make the call. */
1579	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1580
1581	/* Restore variables and guest shadow registers to volatile registers. */
1582	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1583	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1584	TlbState.getActiveRegsWithShadows()
1585	\| RT_BIT_32(idxRegPc)
1586	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1587
1588	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1589	if (!TlbState.fSkip)
1590	{
1591	/* end of TlbMiss - Jump to the done label. */
1592	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1593	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1594
1595	/*
1596	* TlbLookup:
1597	*/
1598	off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1599	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1600	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1601
1602	# ifdef VBOX_WITH_STATISTICS
1603	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1604	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1605	# endif
1606
1607	/*
1608	* TlbDone:
1609	*/
1610	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1611	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1612	}
1613	#else
1614	RT_NOREF(idxLabelTlbMiss);
1615	#endif
1616
1617	/* Jmp back to the start and redo the checks. */
1618	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1619	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1620
1621	/*
1622	* End:
1623	*
1624	* The end.
1625	*/
1626	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1627
1628	if (!TlbState.fSkip)
1629	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1630	else
1631	{
1632	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1633	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1634	iemNativeRegFreeTmp(pReNative, idxRegPc);
1635	if (idxRegCsBase != UINT8_MAX)
1636	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1637	}
1638	return off;
1639	}
1640
1641
1642	#ifdef BODY_CHECK_CS_LIM
1643	/**
1644	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1645	* raising a \#GP(0) if this isn't the case.
1646	*/
1647	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1648	{
1649	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1650	BODY_SET_CUR_INSTR();
1651	BODY_FLUSH_PENDING_WRITES();
1652	BODY_CHECK_CS_LIM(cbInstr);
1653	return off;
1654	}
1655
1656	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1657	{
1658	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1659	LIVENESS_CHECK_CS_LIM(pOutgoing);
1660	RT_NOREF(pCallEntry);
1661	}
1662	#endif
1663
1664
1665	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1666	/**
1667	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1668	* that may have modified them.
1669	*/
1670	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1671	{
1672	PCIEMTB const pTb = pReNative->pTbOrg;
1673	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1674	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1675	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1676	BODY_SET_CUR_INSTR();
1677	BODY_FLUSH_PENDING_WRITES();
1678	BODY_CHECK_CS_LIM(cbInstr);
1679	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1680	return off;
1681	}
1682
1683	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1684	{
1685	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1686	LIVENESS_CHECK_CS_LIM(pOutgoing);
1687	LIVENESS_CHECK_OPCODES(pOutgoing);
1688	RT_NOREF(pCallEntry);
1689	}
1690	#endif
1691
1692
1693	#if defined(BODY_CHECK_OPCODES)
1694	/**
1695	* Built-in function for re-checking opcodes after an instruction that may have
1696	* modified them.
1697	*/
1698	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1699	{
1700	PCIEMTB const pTb = pReNative->pTbOrg;
1701	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1702	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1703	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1704	BODY_SET_CUR_INSTR();
1705	BODY_FLUSH_PENDING_WRITES();
1706	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1707	return off;
1708	}
1709
1710	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1711	{
1712	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1713	LIVENESS_CHECK_OPCODES(pOutgoing);
1714	RT_NOREF(pCallEntry);
1715	}
1716	#endif
1717
1718
1719	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1720	/**
1721	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1722	* checking after an instruction that may have modified them.
1723	*/
1724	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1725	{
1726	PCIEMTB const pTb = pReNative->pTbOrg;
1727	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1728	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1729	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1730	BODY_SET_CUR_INSTR();
1731	BODY_FLUSH_PENDING_WRITES();
1732	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1733	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1734	return off;
1735	}
1736
1737	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1738	{
1739	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1740	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1741	LIVENESS_CHECK_OPCODES(pOutgoing);
1742	RT_NOREF(pCallEntry);
1743	}
1744	#endif
1745
1746
1747	/*
1748	* Post-branching checkers.
1749	*/
1750
1751	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1752	/**
1753	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1754	* after conditional branching within the same page.
1755	*
1756	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1757	*/
1758	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1759	{
1760	PCIEMTB const pTb = pReNative->pTbOrg;
1761	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1762	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1763	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1764	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1765	BODY_SET_CUR_INSTR();
1766	BODY_FLUSH_PENDING_WRITES();
1767	BODY_CHECK_CS_LIM(cbInstr);
1768	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1769	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1770	//LogFunc(("okay\n"));
1771	return off;
1772	}
1773
1774	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1775	{
1776	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1777	LIVENESS_CHECK_CS_LIM(pOutgoing);
1778	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1779	LIVENESS_CHECK_OPCODES(pOutgoing);
1780	RT_NOREF(pCallEntry);
1781	}
1782	#endif
1783
1784
1785	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1786	/**
1787	* Built-in function for checking the PC and checking opcodes after conditional
1788	* branching within the same page.
1789	*
1790	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1791	*/
1792	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1793	{
1794	PCIEMTB const pTb = pReNative->pTbOrg;
1795	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1796	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1797	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1798	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1799	BODY_SET_CUR_INSTR();
1800	BODY_FLUSH_PENDING_WRITES();
1801	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1802	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1803	//LogFunc(("okay\n"));
1804	return off;
1805	}
1806
1807	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1808	{
1809	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1810	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1811	LIVENESS_CHECK_OPCODES(pOutgoing);
1812	RT_NOREF(pCallEntry);
1813	}
1814	#endif
1815
1816
1817	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1818	/**
1819	* Built-in function for checking the PC and checking opcodes and considering
1820	* the need for CS.LIM checking after conditional branching within the same
1821	* page.
1822	*
1823	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1824	*/
1825	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1826	{
1827	PCIEMTB const pTb = pReNative->pTbOrg;
1828	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1829	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1830	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1831	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1832	BODY_SET_CUR_INSTR();
1833	BODY_FLUSH_PENDING_WRITES();
1834	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1835	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1836	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1837	//LogFunc(("okay\n"));
1838	return off;
1839	}
1840
1841	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1842	{
1843	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1844	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1845	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1846	LIVENESS_CHECK_OPCODES(pOutgoing);
1847	RT_NOREF(pCallEntry);
1848	}
1849	#endif
1850
1851
1852	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1853	/**
1854	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1855	* transitioning to a different code page.
1856	*
1857	* The code page transition can either be natural over onto the next page (with
1858	* the instruction starting at page offset zero) or by means of branching.
1859	*
1860	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
1861	*/
1862	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1863	{
1864	PCIEMTB const pTb = pReNative->pTbOrg;
1865	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1866	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1867	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1868	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1869	BODY_SET_CUR_INSTR();
1870	BODY_FLUSH_PENDING_WRITES();
1871	BODY_CHECK_CS_LIM(cbInstr);
1872	Assert(offRange == 0);
1873	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1874	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1875	//LogFunc(("okay\n"));
1876	return off;
1877	}
1878
1879	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1880	{
1881	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1882	LIVENESS_CHECK_CS_LIM(pOutgoing);
1883	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1884	LIVENESS_CHECK_OPCODES(pOutgoing);
1885	RT_NOREF(pCallEntry);
1886	}
1887	#endif
1888
1889
1890	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
1891	/**
1892	* Built-in function for loading TLB and checking opcodes when transitioning to
1893	* a different code page.
1894	*
1895	* The code page transition can either be natural over onto the next page (with
1896	* the instruction starting at page offset zero) or by means of branching.
1897	*
1898	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1899	*/
1900	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
1901	{
1902	PCIEMTB const pTb = pReNative->pTbOrg;
1903	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1904	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1905	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1906	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1907	BODY_SET_CUR_INSTR();
1908	BODY_FLUSH_PENDING_WRITES();
1909	Assert(offRange == 0);
1910	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1911	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1912	//LogFunc(("okay\n"));
1913	return off;
1914	}
1915
1916	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
1917	{
1918	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1919	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1920	LIVENESS_CHECK_OPCODES(pOutgoing);
1921	RT_NOREF(pCallEntry);
1922	}
1923	#endif
1924
1925
1926	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1927	/**
1928	* Built-in function for loading TLB and checking opcodes and considering the
1929	* need for CS.LIM checking when transitioning to a different code page.
1930	*
1931	* The code page transition can either be natural over onto the next page (with
1932	* the instruction starting at page offset zero) or by means of branching.
1933	*
1934	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1935	*/
1936	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1937	{
1938	PCIEMTB const pTb = pReNative->pTbOrg;
1939	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1940	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1941	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1942	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1943	BODY_SET_CUR_INSTR();
1944	BODY_FLUSH_PENDING_WRITES();
1945	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1946	Assert(offRange == 0);
1947	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1948	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1949	//LogFunc(("okay\n"));
1950	return off;
1951	}
1952
1953	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1954	{
1955	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1956	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1957	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1958	LIVENESS_CHECK_OPCODES(pOutgoing);
1959	RT_NOREF(pCallEntry);
1960	}
1961	#endif
1962
1963
1964
1965	/*
1966	* Natural page crossing checkers.
1967	*/
1968
1969	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
1970	/**
1971	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
1972	* both pages when transitioning to a different code page.
1973	*
1974	* This is used when the previous instruction requires revalidation of opcodes
1975	* bytes and the current instruction stries a page boundrary with opcode bytes
1976	* in both the old and new page.
1977	*
1978	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
1979	*/
1980	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1981	{
1982	PCIEMTB const pTb = pReNative->pTbOrg;
1983	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1984	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
1985	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
1986	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
1987	uint32_t const idxRange2 = idxRange1 + 1;
1988	BODY_SET_CUR_INSTR();
1989	BODY_FLUSH_PENDING_WRITES();
1990	BODY_CHECK_CS_LIM(cbInstr);
1991	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
1992	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
1993	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
1994	return off;
1995	}
1996
1997	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1998	{
1999	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2000	LIVENESS_CHECK_CS_LIM(pOutgoing);
2001	LIVENESS_CHECK_OPCODES(pOutgoing);
2002	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2003	RT_NOREF(pCallEntry);
2004	}
2005	#endif
2006
2007
2008	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2009	/**
2010	* Built-in function for loading TLB and checking opcodes on both pages when
2011	* transitioning to a different code page.
2012	*
2013	* This is used when the previous instruction requires revalidation of opcodes
2014	* bytes and the current instruction stries a page boundrary with opcode bytes
2015	* in both the old and new page.
2016	*
2017	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2018	*/
2019	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2020	{
2021	PCIEMTB const pTb = pReNative->pTbOrg;
2022	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2023	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2024	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2025	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2026	uint32_t const idxRange2 = idxRange1 + 1;
2027	BODY_SET_CUR_INSTR();
2028	BODY_FLUSH_PENDING_WRITES();
2029	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2030	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2031	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2032	return off;
2033	}
2034
2035	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2036	{
2037	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2038	LIVENESS_CHECK_OPCODES(pOutgoing);
2039	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2040	RT_NOREF(pCallEntry);
2041	}
2042	#endif
2043
2044
2045	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2046	/**
2047	* Built-in function for loading TLB and checking opcodes on both pages and
2048	* considering the need for CS.LIM checking when transitioning to a different
2049	* code page.
2050	*
2051	* This is used when the previous instruction requires revalidation of opcodes
2052	* bytes and the current instruction stries a page boundrary with opcode bytes
2053	* in both the old and new page.
2054	*
2055	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2056	*/
2057	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2058	{
2059	PCIEMTB const pTb = pReNative->pTbOrg;
2060	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2061	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2062	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2063	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2064	uint32_t const idxRange2 = idxRange1 + 1;
2065	BODY_SET_CUR_INSTR();
2066	BODY_FLUSH_PENDING_WRITES();
2067	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2068	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2069	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2070	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2071	return off;
2072	}
2073
2074	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2075	{
2076	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2077	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2078	LIVENESS_CHECK_OPCODES(pOutgoing);
2079	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2080	RT_NOREF(pCallEntry);
2081	}
2082	#endif
2083
2084
2085	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2086	/**
2087	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2088	* advancing naturally to a different code page.
2089	*
2090	* Only opcodes on the new page is checked.
2091	*
2092	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2093	*/
2094	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2095	{
2096	PCIEMTB const pTb = pReNative->pTbOrg;
2097	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2098	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2099	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2100	//uint32_t const offRange1 = (uint32_t)uParam2;
2101	uint32_t const idxRange2 = idxRange1 + 1;
2102	BODY_SET_CUR_INSTR();
2103	BODY_FLUSH_PENDING_WRITES();
2104	BODY_CHECK_CS_LIM(cbInstr);
2105	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2106	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2107	return off;
2108	}
2109
2110	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2111	{
2112	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2113	LIVENESS_CHECK_CS_LIM(pOutgoing);
2114	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2115	LIVENESS_CHECK_OPCODES(pOutgoing);
2116	RT_NOREF(pCallEntry);
2117	}
2118	#endif
2119
2120
2121	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2122	/**
2123	* Built-in function for loading TLB and checking opcodes when advancing
2124	* naturally to a different code page.
2125	*
2126	* Only opcodes on the new page is checked.
2127	*
2128	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2129	*/
2130	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2131	{
2132	PCIEMTB const pTb = pReNative->pTbOrg;
2133	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2134	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2135	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2136	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2137	uint32_t const idxRange2 = idxRange1 + 1;
2138	BODY_SET_CUR_INSTR();
2139	BODY_FLUSH_PENDING_WRITES();
2140	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2141	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2142	return off;
2143	}
2144
2145	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2146	{
2147	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2148	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2149	LIVENESS_CHECK_OPCODES(pOutgoing);
2150	RT_NOREF(pCallEntry);
2151	}
2152	#endif
2153
2154
2155	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2156	/**
2157	* Built-in function for loading TLB and checking opcodes and considering the
2158	* need for CS.LIM checking when advancing naturally to a different code page.
2159	*
2160	* Only opcodes on the new page is checked.
2161	*
2162	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2163	*/
2164	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2165	{
2166	PCIEMTB const pTb = pReNative->pTbOrg;
2167	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2168	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2169	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2170	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2171	uint32_t const idxRange2 = idxRange1 + 1;
2172	BODY_SET_CUR_INSTR();
2173	BODY_FLUSH_PENDING_WRITES();
2174	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2175	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2176	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2177	return off;
2178	}
2179
2180	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2181	{
2182	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2183	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2184	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2185	LIVENESS_CHECK_OPCODES(pOutgoing);
2186	RT_NOREF(pCallEntry);
2187	}
2188	#endif
2189
2190
2191	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2192	/**
2193	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2194	* advancing naturally to a different code page with first instr at byte 0.
2195	*
2196	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2197	*/
2198	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2199	{
2200	PCIEMTB const pTb = pReNative->pTbOrg;
2201	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2202	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2203	BODY_SET_CUR_INSTR();
2204	BODY_FLUSH_PENDING_WRITES();
2205	BODY_CHECK_CS_LIM(cbInstr);
2206	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2207	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2208	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2209	return off;
2210	}
2211
2212	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2213	{
2214	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2215	LIVENESS_CHECK_CS_LIM(pOutgoing);
2216	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2217	LIVENESS_CHECK_OPCODES(pOutgoing);
2218	RT_NOREF(pCallEntry);
2219	}
2220	#endif
2221
2222
2223	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2224	/**
2225	* Built-in function for loading TLB and checking opcodes when advancing
2226	* naturally to a different code page with first instr at byte 0.
2227	*
2228	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2229	*/
2230	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2231	{
2232	PCIEMTB const pTb = pReNative->pTbOrg;
2233	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2234	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2235	BODY_SET_CUR_INSTR();
2236	BODY_FLUSH_PENDING_WRITES();
2237	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2238	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2239	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2240	return off;
2241	}
2242
2243	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2244	{
2245	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2246	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2247	LIVENESS_CHECK_OPCODES(pOutgoing);
2248	RT_NOREF(pCallEntry);
2249	}
2250	#endif
2251
2252
2253	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2254	/**
2255	* Built-in function for loading TLB and checking opcodes and considering the
2256	* need for CS.LIM checking when advancing naturally to a different code page
2257	* with first instr at byte 0.
2258	*
2259	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2260	*/
2261	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2262	{
2263	PCIEMTB const pTb = pReNative->pTbOrg;
2264	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2265	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2266	BODY_SET_CUR_INSTR();
2267	BODY_FLUSH_PENDING_WRITES();
2268	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2269	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2270	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2271	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2272	return off;
2273	}
2274
2275	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2276	{
2277	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2278	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2279	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2280	LIVENESS_CHECK_OPCODES(pOutgoing);
2281	RT_NOREF(pCallEntry);
2282	}
2283	#endif
2284

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp@ 104933

Download in other formats: