IEMAllN8veRecompBltIn.cpp@ 103778

Last change on this file since 103778 was 103649, checked in by vboxsync, 9 months ago
VMM/IEM: Delay the RIP update for each instruction if possible to save on potentially two instructions for each executed guest instruction of which one is a memory write, bugref:10373
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 96.1 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 103649 2024-03-03 07:11:39Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	//#define IEM_WITH_OPAQUE_DECODER_STATE - need offCurInstrStart access for iemNativeHlpMemCodeNewPageTlbMiss and friends.
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/cpum.h>
38	#include <VBox/vmm/dbgf.h>
39	#include "IEMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/log.h>
42	#include <VBox/err.h>
43	#include <VBox/param.h>
44	#include <iprt/assert.h>
45	#include <iprt/string.h>
46	#if defined(RT_ARCH_AMD64)
47	# include <iprt/x86.h>
48	#elif defined(RT_ARCH_ARM64)
49	# include <iprt/armv8.h>
50	#endif
51
52
53	#include "IEMInline.h"
54	#include "IEMThreadedFunctions.h"
55	#include "IEMN8veRecompiler.h"
56	#include "IEMN8veRecompilerEmit.h"
57	#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61	/*********************************************************************************************************************************
62	* TB Helper Functions *
63	*********************************************************************************************************************************/
64	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
65	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66	#endif
67
68
69	/**
70	* Used by TB code to deal with a TLB miss for a new page.
71	*/
72	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73	{
74	STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
75	pVCpu->iem.s.pbInstrBuf = NULL;
76	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
77	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
78	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
79	if (pVCpu->iem.s.pbInstrBuf)
80	{ /* likely */ }
81	else
82	{
83	IEM_DO_LONGJMP(pVCpu, VINF_IEM_REEXEC_BREAK);
84	}
85	}
86
87
88	/**
89	* Used by TB code to deal with a TLB miss for a new page.
90	*/
91	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
92	{
93	STAM_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
94	pVCpu->iem.s.pbInstrBuf = NULL;
95	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
96	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
97	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
98	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
99	}
100
101
102	/*********************************************************************************************************************************
103	* Builtin functions *
104	*********************************************************************************************************************************/
105
106	/**
107	* Built-in function that does nothing.
108	*
109	* Whether this is called or not can be controlled by the entry in the
110	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
111	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
112	* whether it's the reduced call count in the TBs or the threaded calls flushing
113	* register state.
114	*/
115	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
116	{
117	RT_NOREF(pReNative, pCallEntry);
118	return off;
119	}
120
121	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
122	{
123	pOutgoing = pIncoming;
124	RT_NOREF(pCallEntry);
125	}
126
127
128	/**
129	* Emits for for LogCpuState.
130	*
131	* This shouldn't have any relevant impact on the recompiler state.
132	*/
133	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
134	{
135	#ifdef RT_ARCH_AMD64
136	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
137	/* push rax */
138	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
139	/* push imm32 */
140	pbCodeBuf[off++] = 0x68;
141	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
142	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
143	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
144	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
145	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
146	pbCodeBuf[off++] = X86_OP_REX_W;
147	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
148	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
149	off += sizeof(uint64_t);
150	/* call rax */
151	pbCodeBuf[off++] = 0xff;
152	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
153	/* pop rax */
154	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
155	/* pop rax */
156	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
157	#else
158	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
159	RT_NOREF(pCallEntry);
160	#endif
161
162	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
163	return off;
164	}
165
166	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
167	{
168	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
169	RT_NOREF(pCallEntry);
170	}
171
172
173	/**
174	* Built-in function that calls a C-implemention function taking zero arguments.
175	*/
176	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
177	{
178	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
179	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
180	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
181	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
182	}
183
184	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
185	{
186	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
187	RT_NOREF(pCallEntry);
188	}
189
190
191	/**
192	* Flushes pending writes in preparation of raising an exception or aborting the TB.
193	*/
194	#define BODY_FLUSH_PENDING_WRITES() \
195	off = iemNativeRegFlushPendingWrites(pReNative, off);
196
197
198	/**
199	* Built-in function that checks for pending interrupts that can be delivered or
200	* forced action flags.
201	*
202	* This triggers after the completion of an instruction, so EIP is already at
203	* the next instruction. If an IRQ or important FF is pending, this will return
204	* a non-zero status that stops TB execution.
205	*/
206	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
207	{
208	RT_NOREF(pCallEntry);
209
210	BODY_FLUSH_PENDING_WRITES();
211
212	/* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
213	and I'm too lazy to create a 'Fixed' version of that one. */
214	uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
215	UINT32_MAX, pReNative->uCheckIrqSeqNo++);
216
217	uint32_t const idxLabelReturnBreak = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ReturnBreak);
218
219	/* Again, we need to load the extended EFLAGS before we actually need them
220	in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
221	loaded them inside the check, as the shadow state would not be correct
222	when the code branches before the load. Ditto PC. */
223	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
224	kIemNativeGstRegUse_ReadOnly);
225
226	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
227
228	uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
229
230	/*
231	* Start by checking the local forced actions of the EMT we're on for IRQs
232	* and other FFs that needs servicing.
233	*/
234	/** @todo this isn't even close to the NMI and interrupt conditions in EM! */
235	/* Load FFs in to idxTmpReg and AND with all relevant flags. */
236	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
237	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
238	VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
239	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
240	\| VMCPU_FF_TLB_FLUSH
241	\| VMCPU_FF_UNHALT ),
242	true /fSetFlags/);
243	/* If we end up with ZERO in idxTmpReg there is nothing to do.*/
244	uint32_t const offFixupJumpToVmCheck1 = off;
245	off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
246
247	/* Some relevant FFs are set, but if's only APIC or/and PIC being set,
248	these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
249	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
250	~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC), true /fSetFlags/);
251	/* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
252	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);
253
254	/* So, it's only interrupt releated FFs and we need to see if IRQs are being
255	suppressed by the CPU or not. */
256	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
257	off = iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
258	idxLabelReturnBreak);
259
260	/* We've got shadow flags set, so we must check that the PC they are valid
261	for matches our current PC value. */
262	/** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
263	* a register. */
264	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
265	off = iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, idxTmpReg, idxPcReg, idxLabelReturnBreak);
266
267	/*
268	* Now check the force flags of the VM.
269	*/
270	iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
271	iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
272	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
273	off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
274	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /fSetFlags/);
275	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelReturnBreak);
276
277	/** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
278
279	/*
280	* We're good, no IRQs or FFs pending.
281	*/
282	iemNativeRegFreeTmp(pReNative, idxTmpReg);
283	iemNativeRegFreeTmp(pReNative, idxEflReg);
284	iemNativeRegFreeTmp(pReNative, idxPcReg);
285
286	return off;
287	}
288
289	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
290	{
291	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
292	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
293	RT_NOREF(pCallEntry);
294	}
295
296
297	/**
298	* Built-in function checks if IEMCPU::fExec has the expected value.
299	*/
300	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
301	{
302	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
303	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
304
305	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
306	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
307	off = iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
308	kIemNativeLabelType_ReturnBreak);
309	iemNativeRegFreeTmp(pReNative, idxTmpReg);
310
311	/* Maintain the recompiler fExec state. */
312	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
313	return off;
314	}
315
316	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
317	{
318	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
319	RT_NOREF(pCallEntry);
320	}
321
322
323	/**
324	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
325	*/
326	/** @todo Optimize this, so we don't set the same value more than once. Just
327	* needs some tracking. */
328	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
329	# define BODY_SET_CUR_INSTR() \
330	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
331	#else
332	# define BODY_SET_CUR_INSTR() ((void)0)
333	#endif
334
335
336	/**
337	* Macro that emits the 16/32-bit CS.LIM check.
338	*/
339	#define BODY_CHECK_CS_LIM(a_cbInstr) \
340	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
341
342	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
343	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
344
345	DECL_FORCE_INLINE(uint32_t)
346	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
347	{
348	Assert(cbInstr > 0);
349	Assert(cbInstr < 16);
350	#ifdef VBOX_STRICT
351	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
352	#endif
353
354	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
355	Assert(pReNative->Core.offPc == 0);
356	#endif
357
358	/*
359	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
360	* a temporary register for calculating the last address of the instruction.
361	*
362	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
363	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
364	* that last updated EIP here checked it already, and that we're therefore
365	* safe in the 32-bit wrap-around scenario to only check that the last byte
366	* is within CS.LIM. In the case of instruction-by-instruction advancing
367	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
368	* must be using 4KB granularity and the previous instruction was fine.
369	*/
370	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
371	kIemNativeGstRegUse_ReadOnly);
372	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
373	kIemNativeGstRegUse_ReadOnly);
374	#ifdef RT_ARCH_AMD64
375	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
376	#elif defined(RT_ARCH_ARM64)
377	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
378	#else
379	# error "Port me"
380	#endif
381
382	if (cbInstr != 1)
383	{
384	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
385
386	/*
387	* 1. idxRegTmp = idxRegPc + cbInstr;
388	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
389	*/
390	#ifdef RT_ARCH_AMD64
391	/* 1. lea tmp32, [Pc + cbInstr - 1] */
392	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
393	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
394	pbCodeBuf[off++] = 0x8d;
395	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
396	if ((idxRegPc & 7) == X86_GREG_xSP)
397	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
398	pbCodeBuf[off++] = cbInstr - 1;
399
400	/* 2. cmp tmp32(r), CsLim(r/m). */
401	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
402	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
403	pbCodeBuf[off++] = 0x3b;
404	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
405
406	#elif defined(RT_ARCH_ARM64)
407	/* 1. add tmp32, Pc, #cbInstr-1 */
408	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
409	/* 2. cmp tmp32, CsLim */
410	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
411	false /f64Bit/, true /fSetFlags/);
412
413	#endif
414	iemNativeRegFreeTmp(pReNative, idxRegTmp);
415	}
416	else
417	{
418	/*
419	* Here we can skip step 1 and compare PC and CS.LIM directly.
420	*/
421	#ifdef RT_ARCH_AMD64
422	/* 2. cmp eip(r), CsLim(r/m). */
423	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
424	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
425	pbCodeBuf[off++] = 0x3b;
426	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
427
428	#elif defined(RT_ARCH_ARM64)
429	/* 2. cmp Pc, CsLim */
430	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
431	false /f64Bit/, true /fSetFlags/);
432
433	#endif
434	}
435
436	/* 3. Jump if greater. */
437	off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_RaiseGp0);
438
439	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
440	iemNativeRegFreeTmp(pReNative, idxRegPc);
441	return off;
442	}
443
444
445	/**
446	* Macro that considers whether we need CS.LIM checking after a branch or
447	* crossing over to a new page.
448	*/
449	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
450	RT_NOREF(a_cbInstr); \
451	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
452
453	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
454	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
455	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
456
457	DECL_FORCE_INLINE(uint32_t)
458	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
459	{
460	#ifdef VBOX_STRICT
461	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
462	#endif
463
464	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
465	Assert(pReNative->Core.offPc == 0);
466	#endif
467
468	/*
469	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
470	* exactly:
471	*
472	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
473	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
474	* return fRet;
475	* return fRet \| IEMTB_F_CS_LIM_CHECKS;
476	*
477	*
478	* We need EIP, CS.LIM and CS.BASE here.
479	*/
480
481	/* Calculate the offFromLim first: */
482	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
483	kIemNativeGstRegUse_ReadOnly);
484	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
485	kIemNativeGstRegUse_ReadOnly);
486	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
487
488	#ifdef RT_ARCH_ARM64
489	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
490	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
491	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
492	#else
493	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
494	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
495	#endif
496
497	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
498	iemNativeRegFreeTmp(pReNative, idxRegPc);
499
500	/* Calculate the threshold level (right side). */
501	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
502	kIemNativeGstRegUse_ReadOnly);
503	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
504
505	#ifdef RT_ARCH_ARM64
506	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
507	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
508	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
509	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
510	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
511	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
512	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
513
514	#else
515	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
516	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
517	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
518	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
519	#endif
520
521	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
522
523	/* Compare the two and jump out if we're too close to the limit. */
524	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
525	off = iemNativeEmitJlToNewLabel(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
526
527	iemNativeRegFreeTmp(pReNative, idxRegRight);
528	iemNativeRegFreeTmp(pReNative, idxRegLeft);
529	return off;
530	}
531
532
533
534	/**
535	* Macro that implements opcode (re-)checking.
536	*/
537	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
538	RT_NOREF(a_cbInstr); \
539	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
540
541	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
542
543	#if 0 /* debugging aid */
544	bool g_fBpOnObsoletion = false;
545	# define BP_ON_OBSOLETION g_fBpOnObsoletion
546	#else
547	# define BP_ON_OBSOLETION 0
548	#endif
549
550	DECL_FORCE_INLINE(uint32_t)
551	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
552	{
553	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
554	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
555	#ifdef VBOX_STRICT
556	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
557	#endif
558
559	uint32_t const idxLabelObsoleteTb = iemNativeLabelCreate(pReNative, kIemNativeLabelType_ObsoleteTb);
560
561	/*
562	* Where to start and how much to compare.
563	*
564	* Looking at the ranges produced when r160746 was running a DOS VM with TB
565	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
566	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
567	*
568	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
569	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
570	*/
571	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
572	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
573	Assert(cbLeft > 0);
574	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
575	uint32_t offConsolidatedJump = UINT32_MAX;
576
577	#ifdef RT_ARCH_AMD64
578	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
579	completely inlined, for larger we use REPE CMPS. */
580	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
581	pbCodeBuf[off++] = a_bOpcode; \
582	Assert(offPage < 127); \
583	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
584	pbCodeBuf[off++] = RT_BYTE1(offPage); \
585	} while (0)
586
587	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
588	if (offConsolidatedJump != UINT32_MAX) \
589	{ \
590	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
591	Assert(offDisp >= -128); \
592	pbCodeBuf[off++] = 0x75; /* jnz near */ \
593	pbCodeBuf[off++] = (uint8_t)offDisp; \
594	} \
595	else \
596	{ \
597	pbCodeBuf[off++] = 0x74; /* jz near +5 */ \
598	pbCodeBuf[off++] = 0x05 + BP_ON_OBSOLETION; \
599	offConsolidatedJump = off; \
600	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
601	pbCodeBuf[off++] = 0xe9; /* jmp rel32 */ \
602	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_Rel32, -4); \
603	pbCodeBuf[off++] = 0x00; \
604	pbCodeBuf[off++] = 0x00; \
605	pbCodeBuf[off++] = 0x00; \
606	pbCodeBuf[off++] = 0x00; \
607	} \
608	} while (0)
609
610	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
611	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
612	pbCodeBuf[off++] = *pbOpcodes++; \
613	pbCodeBuf[off++] = *pbOpcodes++; \
614	pbCodeBuf[off++] = *pbOpcodes++; \
615	pbCodeBuf[off++] = *pbOpcodes++; \
616	cbLeft -= 4; \
617	offPage += 4; \
618	CHECK_OPCODES_CMP_JMP(); \
619	} while (0)
620
621	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
622	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
623	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
624	pbCodeBuf[off++] = *pbOpcodes++; \
625	pbCodeBuf[off++] = *pbOpcodes++; \
626	cbLeft -= 2; \
627	offPage += 2; \
628	CHECK_OPCODES_CMP_JMP(); \
629	} while (0)
630
631	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
632	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
633	pbCodeBuf[off++] = *pbOpcodes++; \
634	cbLeft -= 1; \
635	offPage += 1; \
636	CHECK_OPCODES_CMP_JMP(); \
637	} while (0)
638
639	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
640	if (a_bPrefix) \
641	pbCodeBuf[off++] = (a_bPrefix); \
642	pbCodeBuf[off++] = (a_bOpcode); \
643	CHECK_OPCODES_CMP_JMP(); \
644	cbLeft -= (a_cbToSubtract); \
645	} while (0)
646
647	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
648	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
649	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
650	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
651	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
652	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
653	} while (0)
654
655	if (cbLeft <= 24)
656	{
657	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
658	( RT_BIT_32(X86_GREG_xAX)
659	\| RT_BIT_32(X86_GREG_xCX)
660	\| RT_BIT_32(X86_GREG_xDX)
661	\| RT_BIT_32(X86_GREG_xBX)
662	\| RT_BIT_32(X86_GREG_xSI)
663	\| RT_BIT_32(X86_GREG_xDI))
664	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
665	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
666	if (offPage >= 128 - cbLeft)
667	{
668	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
669	offPage &= 3;
670	}
671
672	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 87 */);
673
674	if (cbLeft > 8)
675	switch (offPage & 3)
676	{
677	case 0:
678	break;
679	case 1: /* cost: 6 + 8 = 14 */
680	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
681	RT_FALL_THRU();
682	case 2: /* cost: 8 */
683	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
684	break;
685	case 3: /* cost: 6 */
686	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
687	break;
688	}
689
690	while (cbLeft >= 4)
691	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
692
693	if (cbLeft >= 2)
694	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
695	if (cbLeft)
696	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
697
698	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
699	iemNativeRegFreeTmp(pReNative, idxRegTmp);
700	}
701	else
702	{
703	/* RDI = &pbInstrBuf[offPage] */
704	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
705	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
706	if (offPage != 0)
707	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
708
709	/* RSI = pbOpcodes */
710	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
711	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
712
713	/* RCX = counts. */
714	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
715
716	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 35/);
717
718	/** @todo profile and optimize this further. Maybe an idea to align by
719	* offPage if the two cannot be reconsidled. */
720	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
721	switch (offPage & 7) /* max cost: 10 */
722	{
723	case 0:
724	break;
725	case 1: /* cost: 3+4+3 = 10 */
726	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
727	RT_FALL_THRU();
728	case 2: /* cost: 4+3 = 7 */
729	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
730	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
731	break;
732	case 3: /* cost: 3+3 = 6 */
733	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
734	RT_FALL_THRU();
735	case 4: /* cost: 3 */
736	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
737	break;
738	case 5: /* cost: 3+4 = 7 */
739	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
740	RT_FALL_THRU();
741	case 6: /* cost: 4 */
742	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
743	break;
744	case 7: /* cost: 3 */
745	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
746	break;
747	}
748
749	/* Compare qwords: */
750	uint32_t const cQWords = cbLeft >> 3;
751	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
752
753	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
754	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
755	cbLeft &= 7;
756
757	if (cbLeft & 4)
758	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
759	if (cbLeft & 2)
760	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
761	if (cbLeft & 1)
762	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
763
764	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
765	iemNativeRegFreeTmp(pReNative, idxRegCx);
766	iemNativeRegFreeTmp(pReNative, idxRegSi);
767	iemNativeRegFreeTmp(pReNative, idxRegDi);
768	}
769
770	#elif defined(RT_ARCH_ARM64)
771	/* We need pbInstrBuf in a register, whatever we do. */
772	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
773	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
774
775	/* We also need at least one more register for holding bytes & words we
776	load via pbInstrBuf. */
777	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
778
779	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
780
781	/* One byte compare can be done with the opcode byte as an immediate. We'll
782	do this to uint16_t align src1. */
783	bool fPendingJmp = RT_BOOL(offPage & 1);
784	if (fPendingJmp)
785	{
786	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
787	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
788	offPage += 1;
789	cbLeft -= 1;
790	}
791
792	if (cbLeft > 0)
793	{
794	/* We need a register for holding the opcode bytes we're comparing with,
795	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
796	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
797
798	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
799	if ((offPage & 3) && cbLeft >= 2)
800	{
801	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
802	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
803	if (fPendingJmp)
804	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
805	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
806	else
807	{
808	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
809	fPendingJmp = true;
810	}
811	pbOpcodes += 2;
812	offPage += 2;
813	cbLeft -= 2;
814	}
815
816	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
817	if ((offPage & 7) && cbLeft >= 4)
818	{
819	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
820	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
821	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
822	if (fPendingJmp)
823	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
824	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
825	else
826	{
827	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
828	fPendingJmp = true;
829	}
830	pbOpcodes += 4;
831	offPage += 4;
832	cbLeft -= 4;
833	}
834
835	/*
836	* If we've got 16 bytes or more left, switch to memcmp-style.
837	*/
838	if (cbLeft >= 16)
839	{
840	/* We need a pointer to the copy of the original opcode bytes. */
841	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
842	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
843
844	/* If there are more than 32 bytes to compare we create a loop, for
845	which we'll need a loop register. */
846	if (cbLeft >= 64)
847	{
848	if (fPendingJmp)
849	{
850	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
851	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
852	fPendingJmp = false;
853	}
854
855	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
856	uint16_t const cLoops = cbLeft / 32;
857	cbLeft = cbLeft % 32;
858	pbOpcodes += cLoops * 32;
859	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
860
861	if (offPage != 0) /** @todo optimize out this instruction. */
862	{
863	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
864	offPage = 0;
865	}
866
867	uint32_t const offLoopStart = off;
868	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
869	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
870	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
871
872	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
873	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
874	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
875	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
876
877	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
878	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
879	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
880	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
881
882	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
883	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
884	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
885	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
886
887	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
888	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
889
890	/* Advance and loop. */
891	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
892	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
893	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
894	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
895
896	iemNativeRegFreeTmp(pReNative, idxRegLoop);
897	}
898
899	/* Deal with any remaining dwords (uint64_t). There can be up to
900	three if we looped and four if we didn't. */
901	uint32_t offSrc2 = 0;
902	while (cbLeft >= 8)
903	{
904	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
905	idxRegSrc1Ptr, offPage / 8);
906	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
907	idxRegSrc2Ptr, offSrc2 / 8);
908	if (fPendingJmp)
909	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
910	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
911	else
912	{
913	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
914	fPendingJmp = true;
915	}
916	pbOpcodes += 8;
917	offPage += 8;
918	offSrc2 += 8;
919	cbLeft -= 8;
920	}
921
922	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
923	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
924	}
925	/*
926	* Otherwise, we compare with constants and merge with the general mop-up.
927	*/
928	else
929	{
930	while (cbLeft >= 8)
931	{
932	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
933	offPage / 8);
934	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
935	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
936	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
937	if (fPendingJmp)
938	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
939	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
940	else
941	{
942	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
943	fPendingJmp = true;
944	}
945	pbOpcodes += 8;
946	offPage += 8;
947	cbLeft -= 8;
948	}
949	/* max cost thus far: 21 */
950	}
951
952	/* Deal with any remaining bytes (7 or less). */
953	Assert(cbLeft < 8);
954	if (cbLeft >= 4)
955	{
956	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
957	offPage / 4);
958	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
959	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
960	if (fPendingJmp)
961	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
962	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
963	else
964	{
965	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
966	fPendingJmp = true;
967	}
968	pbOpcodes += 4;
969	offPage += 4;
970	cbLeft -= 4;
971
972	}
973
974	if (cbLeft >= 2)
975	{
976	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
977	offPage / 2);
978	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
979	if (fPendingJmp)
980	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
981	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
982	else
983	{
984	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
985	fPendingJmp = true;
986	}
987	pbOpcodes += 2;
988	offPage += 2;
989	cbLeft -= 2;
990	}
991
992	if (cbLeft > 0)
993	{
994	Assert(cbLeft == 1);
995	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
996	if (fPendingJmp)
997	{
998	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
999	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1000	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1001	}
1002	else
1003	{
1004	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
1005	fPendingJmp = true;
1006	}
1007	pbOpcodes += 1;
1008	offPage += 1;
1009	cbLeft -= 1;
1010	}
1011
1012	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1013	}
1014	Assert(cbLeft == 0);
1015
1016	/*
1017	* Finally, the branch on difference.
1018	*/
1019	if (fPendingJmp)
1020	{
1021	iemNativeAddFixup(pReNative, off, idxLabelObsoleteTb, kIemNativeFixupType_RelImm19At5);
1022	pu32CodeBuf[off++] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, 0);
1023	}
1024	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump, idxLabelObsoleteTb);
1025
1026	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1027	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1028	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1029	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1030
1031	#else
1032	# error "Port me"
1033	#endif
1034	return off;
1035	}
1036
1037
1038	/** Duplicated in IEMAllThrdFuncsBltIn.cpp. */
1039	DECL_FORCE_INLINE(RTGCPHYS) iemTbGetRangePhysPageAddr(PCIEMTB pTb, uint8_t idxRange)
1040	{
1041	Assert(idxRange < RT_MIN(pTb->cRanges, RT_ELEMENTS(pTb->aRanges)));
1042	uint8_t const idxPage = pTb->aRanges[idxRange].idxPhysPage;
1043	Assert(idxPage <= RT_ELEMENTS(pTb->aGCPhysPages));
1044	if (idxPage == 0)
1045	return pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1046	Assert(!(pTb->aGCPhysPages[idxPage - 1] & GUEST_PAGE_OFFSET_MASK));
1047	return pTb->aGCPhysPages[idxPage - 1];
1048	}
1049
1050
1051	/**
1052	* Macro that implements PC check after a conditional branch.
1053	*/
1054	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1055	RT_NOREF(a_cbInstr); \
1056	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1057
1058	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1059	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1060	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1061	else do { } while (0)
1062
1063	DECL_FORCE_INLINE(uint32_t)
1064	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1065	uint8_t idxRange, uint16_t offRange)
1066	{
1067	#ifdef VBOX_STRICT
1068	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1069	#endif
1070
1071	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1072	Assert(pReNative->Core.offPc == 0);
1073	#endif
1074
1075	/*
1076	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1077	* constant for us here.
1078	*
1079	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1080	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1081	* it, and as long as we don't implement code TLB reload code here there is
1082	* no point in checking that the TLB data we're using is still valid.
1083	*
1084	* What we to do is.
1085	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1086	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1087	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1088	* we're in the wrong spot and need to find a new TB.
1089	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1090	* GCPhysRangePageWithOffset constant mentioned above.
1091	*
1092	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1093	* in 64-bit code or flat 32-bit.
1094	*/
1095
1096	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1097	the temp register, so we don't accidentally clobber something we'll be
1098	needing again immediately. This is why we get idxRegCsBase here. */
1099	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1100	kIemNativeGstRegUse_ReadOnly);
1101	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1102	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1103	kIemNativeGstRegUse_ReadOnly);
1104
1105	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1106
1107	#ifdef VBOX_STRICT
1108	/* Do assertions before idxRegTmp contains anything. */
1109	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1110	# ifdef RT_ARCH_AMD64
1111	{
1112	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1113	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1114	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1115	{
1116	/* cmp r/m64, imm8 */
1117	pbCodeBuf[off++] = X86_OP_REX_W;
1118	pbCodeBuf[off++] = 0x83;
1119	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1120	pbCodeBuf[off++] = 0;
1121	/* je rel8 */
1122	pbCodeBuf[off++] = 0x74;
1123	pbCodeBuf[off++] = 1;
1124	/* int3 */
1125	pbCodeBuf[off++] = 0xcc;
1126
1127	}
1128
1129	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1130	/* test r/m64, imm32 */
1131	pbCodeBuf[off++] = X86_OP_REX_W;
1132	pbCodeBuf[off++] = 0xf7;
1133	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1134	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1135	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1136	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1137	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1138	/* jz rel8 */
1139	pbCodeBuf[off++] = 0x74;
1140	pbCodeBuf[off++] = 1;
1141	/* int3 */
1142	pbCodeBuf[off++] = 0xcc;
1143	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1144	}
1145	# else
1146
1147	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1148	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1149	{
1150	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1151	# ifdef RT_ARCH_ARM64
1152	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1153	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1154	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1155	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1156	# else
1157	# error "Port me!"
1158	# endif
1159	}
1160	# endif
1161
1162	#endif /* VBOX_STRICT */
1163
1164	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1165	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1166	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1167	{
1168	#ifdef RT_ARCH_ARM64
1169	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1170	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1171	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1172	#else
1173	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1174	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1175	#endif
1176	}
1177	else
1178	{
1179	#ifdef RT_ARCH_ARM64
1180	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1181	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1182	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1183	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1184	#else
1185	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1186	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1187	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1188	#endif
1189	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1190	}
1191	iemNativeRegFreeTmp(pReNative, idxRegPc);
1192
1193	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1194	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1195	off = iemNativeEmitJaToNewLabel(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1196
1197	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1198	#ifdef RT_ARCH_AMD64
1199	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1200	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1201	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1202	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1203	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1204
1205	#elif defined(RT_ARCH_ARM64)
1206	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1207
1208	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1209	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1210	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1211	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1212
1213	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1214	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1215	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1216	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1217	# endif
1218	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1219	#else
1220	# error "Port me"
1221	#endif
1222
1223	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1224	\| pTb->aRanges[idxRange].offPhysPage)
1225	+ offRange;
1226	off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1227	kIemNativeLabelType_CheckBranchMiss);
1228
1229	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1230	return off;
1231	}
1232
1233
1234	/**
1235	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1236	* instruction crossing into a new page.
1237	*
1238	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1239	*/
1240	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1241	RT_NOREF(a_cbInstr); \
1242	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1243
1244	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1245	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1246	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1247	else do { } while (0)
1248
1249	DECL_FORCE_INLINE(uint32_t)
1250	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1251	{
1252	#ifdef VBOX_STRICT
1253	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1254	#endif
1255
1256	/*
1257	* Define labels and allocate the register for holding the GCPhys of the new page.
1258	*/
1259	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1260	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1261	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1262	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1263	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1264	: UINT32_MAX;
1265
1266	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1267
1268	/*
1269	* Jump to the TLB lookup code.
1270	*/
1271	if (!TlbState.fSkip)
1272	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1273
1274	/*
1275	* TlbMiss:
1276	*
1277	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1278	*/
1279	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1280
1281	/* Save variables in volatile registers. */
1282	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1283	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1284
1285	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1286	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1287
1288	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1289	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1290
1291	/* Done setting up parameters, make the call. */
1292	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1293
1294	/* Move the result to the right register. */
1295	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1296	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1297
1298	/* Restore variables and guest shadow registers to volatile registers. */
1299	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1300	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1301
1302	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1303	if (!TlbState.fSkip)
1304	{
1305	/* end of TlbMiss - Jump to the done label. */
1306	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1307	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1308
1309	/*
1310	* TlbLookup:
1311	*/
1312	off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1313	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1314	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1315	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1316
1317	# ifdef VBOX_WITH_STATISTICS
1318	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1319	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1320	# endif
1321
1322	/*
1323	* TlbDone:
1324	*/
1325	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1326	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1327	}
1328	#else
1329	RT_NOREF(idxLabelTlbMiss);
1330	#endif
1331
1332	/*
1333	* Now check the physical address of the page matches the expected one.
1334	*/
1335	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1336	off = iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1337	kIemNativeLabelType_ObsoleteTb);
1338
1339	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1340	return off;
1341	}
1342
1343
1344	/**
1345	* Macro that implements TLB loading and updating pbInstrBuf updating when
1346	* branching or when crossing a page on an instruction boundrary.
1347	*
1348	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1349	* it is an inter-page branch and also check the page offset.
1350	*
1351	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1352	*/
1353	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1354	RT_NOREF(a_cbInstr); \
1355	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1356
1357	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1358	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1359	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1360	else do { } while (0)
1361
1362	DECL_FORCE_INLINE(uint32_t)
1363	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1364	{
1365	#ifdef VBOX_STRICT
1366	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1367	#endif
1368
1369	BODY_FLUSH_PENDING_WRITES();
1370
1371	/*
1372	* Define labels and allocate the register for holding the GCPhys of the new page.
1373	*/
1374	uint32_t const idxLabelCheckBranchMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckBranchMiss);
1375	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1376	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1377	\| pTb->aRanges[idxRange].offPhysPage;
1378
1379	/*
1380	*
1381	* First check if RIP is within the current code.
1382	*
1383	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1384	* difference is what we do when stuff doesn't match up.
1385	*
1386	* What we to do is.
1387	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1388	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1389	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1390	* we need to retranslate RIP via the TLB.
1391	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1392	* GCPhysRangePageWithOffset constant mentioned above.
1393	*
1394	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1395	* in 64-bit code or flat 32-bit.
1396	*
1397	*/
1398
1399	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1400	the temp register, so we don't accidentally clobber something we'll be
1401	needing again immediately. This is why we get idxRegCsBase here.
1402	Update: We share registers with the TlbState, as the TLB code path has
1403	little in common with the rest of the code. */
1404	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1405	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1406	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1407	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1408	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1409	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1410	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1411	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1412
1413	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1414	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1415	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1416
1417	#ifdef VBOX_STRICT
1418	/* Do assertions before idxRegTmp contains anything. */
1419	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1420	# ifdef RT_ARCH_AMD64
1421	{
1422	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1423	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1424	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1425	{
1426	/* cmp r/m64, imm8 */
1427	pbCodeBuf[off++] = X86_OP_REX_W;
1428	pbCodeBuf[off++] = 0x83;
1429	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1430	pbCodeBuf[off++] = 0;
1431	/* je rel8 */
1432	pbCodeBuf[off++] = 0x74;
1433	pbCodeBuf[off++] = 1;
1434	/* int3 */
1435	pbCodeBuf[off++] = 0xcc;
1436
1437	}
1438
1439	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1440	/* test r/m64, imm32 */
1441	pbCodeBuf[off++] = X86_OP_REX_W;
1442	pbCodeBuf[off++] = 0xf7;
1443	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1444	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1445	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1446	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1447	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1448	/* jz rel8 */
1449	pbCodeBuf[off++] = 0x74;
1450	pbCodeBuf[off++] = 1;
1451	/* int3 */
1452	pbCodeBuf[off++] = 0xcc;
1453	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1454	}
1455	# else
1456
1457	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1458	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1459	{
1460	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1461	# ifdef RT_ARCH_ARM64
1462	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1463	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1464	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1465	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1466	# else
1467	# error "Port me!"
1468	# endif
1469	}
1470	# endif
1471
1472	#endif /* VBOX_STRICT */
1473
1474	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1475	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1476	2nd register to check if we've looped more than once already.*/
1477	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1478
1479	uint32_t const offLabelRedoChecks = off;
1480
1481	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1482	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1483	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1484	{
1485	#ifdef RT_ARCH_ARM64
1486	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1487	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1488	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1489	#else
1490	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1491	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1492	#endif
1493	}
1494	else
1495	{
1496	#ifdef RT_ARCH_ARM64
1497	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1498	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1499	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1500	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1501	#else
1502	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1503	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1504	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1505	#endif
1506	}
1507
1508	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1509	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1510	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1511	uint32_t const offFixedJumpToTlbLoad = off;
1512	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1513
1514	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1515	#ifdef RT_ARCH_AMD64
1516	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1517	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1518	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1519	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1520	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1521
1522	#elif defined(RT_ARCH_ARM64)
1523
1524	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1525	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1526	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1527	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1528
1529	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1530	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1531	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1532	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1533	# endif
1534	#else
1535	# error "Port me"
1536	#endif
1537
1538	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1539
1540	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1541	careful and avoid implicit temporary register usage here.
1542
1543	Unlike the threaded version of this code, we do not obsolete TBs here to
1544	reduce the code size and because indirect calls may legally end at the
1545	same offset in two different pages depending on the program state. */
1546	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1547	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1548	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1549	off = iemNativeEmitJnzToLabel(pReNative, off, idxLabelCheckBranchMiss);
1550	uint32_t const offFixedJumpToEnd = off;
1551	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1552
1553	/*
1554	* TlbLoad:
1555	*
1556	* First we try to go via the TLB.
1557	*/
1558	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1559
1560	/* Check that we haven't been here before. */
1561	off = iemNativeEmitTestIfGprIsNotZeroAndJmpToLabel(pReNative, off, idxRegTmp2, false /f64Bit/, idxLabelCheckBranchMiss);
1562
1563	/* Jump to the TLB lookup code. */
1564	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1565	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1566	: UINT32_MAX;
1567	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1568	if (!TlbState.fSkip)
1569	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1570
1571	/*
1572	* TlbMiss:
1573	*
1574	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1575	*/
1576	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1577	RT_NOREF(idxLabelTlbMiss);
1578
1579	/* Save variables in volatile registers. */
1580	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1581	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1582	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1583
1584	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1585	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1586
1587	/* Done setting up parameters, make the call. */
1588	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1589
1590	/* Restore variables and guest shadow registers to volatile registers. */
1591	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1592	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1593	TlbState.getActiveRegsWithShadows()
1594	\| RT_BIT_32(idxRegPc)
1595	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1596
1597	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1598	if (!TlbState.fSkip)
1599	{
1600	/* end of TlbMiss - Jump to the done label. */
1601	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1602	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1603
1604	/*
1605	* TlbLookup:
1606	*/
1607	off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1608	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1609	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1610
1611	# ifdef VBOX_WITH_STATISTICS
1612	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1613	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1614	# endif
1615
1616	/*
1617	* TlbDone:
1618	*/
1619	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1620	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1621	}
1622	#else
1623	RT_NOREF(idxLabelTlbMiss);
1624	#endif
1625
1626	/* Jmp back to the start and redo the checks. */
1627	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1628	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1629
1630	/*
1631	* End:
1632	*
1633	* The end.
1634	*/
1635	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1636
1637	if (!TlbState.fSkip)
1638	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1639	else
1640	{
1641	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1642	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1643	iemNativeRegFreeTmp(pReNative, idxRegPc);
1644	if (idxRegCsBase != UINT8_MAX)
1645	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1646	}
1647	return off;
1648	}
1649
1650
1651	#ifdef BODY_CHECK_CS_LIM
1652	/**
1653	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1654	* raising a \#GP(0) if this isn't the case.
1655	*/
1656	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1657	{
1658	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1659	BODY_SET_CUR_INSTR();
1660	BODY_FLUSH_PENDING_WRITES();
1661	BODY_CHECK_CS_LIM(cbInstr);
1662	return off;
1663	}
1664
1665	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1666	{
1667	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1668	LIVENESS_CHECK_CS_LIM(pOutgoing);
1669	RT_NOREF(pCallEntry);
1670	}
1671	#endif
1672
1673
1674	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1675	/**
1676	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1677	* that may have modified them.
1678	*/
1679	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1680	{
1681	PCIEMTB const pTb = pReNative->pTbOrg;
1682	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1683	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1684	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1685	BODY_SET_CUR_INSTR();
1686	BODY_FLUSH_PENDING_WRITES();
1687	BODY_CHECK_CS_LIM(cbInstr);
1688	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1689	return off;
1690	}
1691
1692	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1693	{
1694	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1695	LIVENESS_CHECK_CS_LIM(pOutgoing);
1696	LIVENESS_CHECK_OPCODES(pOutgoing);
1697	RT_NOREF(pCallEntry);
1698	}
1699	#endif
1700
1701
1702	#if defined(BODY_CHECK_OPCODES)
1703	/**
1704	* Built-in function for re-checking opcodes after an instruction that may have
1705	* modified them.
1706	*/
1707	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1708	{
1709	PCIEMTB const pTb = pReNative->pTbOrg;
1710	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1711	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1712	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1713	BODY_SET_CUR_INSTR();
1714	BODY_FLUSH_PENDING_WRITES();
1715	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1716	return off;
1717	}
1718
1719	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1720	{
1721	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1722	LIVENESS_CHECK_OPCODES(pOutgoing);
1723	RT_NOREF(pCallEntry);
1724	}
1725	#endif
1726
1727
1728	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1729	/**
1730	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1731	* checking after an instruction that may have modified them.
1732	*/
1733	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1734	{
1735	PCIEMTB const pTb = pReNative->pTbOrg;
1736	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1737	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1738	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1739	BODY_SET_CUR_INSTR();
1740	BODY_FLUSH_PENDING_WRITES();
1741	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1742	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1743	return off;
1744	}
1745
1746	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1747	{
1748	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1749	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1750	LIVENESS_CHECK_OPCODES(pOutgoing);
1751	RT_NOREF(pCallEntry);
1752	}
1753	#endif
1754
1755
1756	/*
1757	* Post-branching checkers.
1758	*/
1759
1760	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1761	/**
1762	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1763	* after conditional branching within the same page.
1764	*
1765	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1766	*/
1767	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1768	{
1769	PCIEMTB const pTb = pReNative->pTbOrg;
1770	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1771	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1772	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1773	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1774	BODY_SET_CUR_INSTR();
1775	BODY_FLUSH_PENDING_WRITES();
1776	BODY_CHECK_CS_LIM(cbInstr);
1777	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1778	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1779	//LogFunc(("okay\n"));
1780	return off;
1781	}
1782
1783	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1784	{
1785	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1786	LIVENESS_CHECK_CS_LIM(pOutgoing);
1787	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1788	LIVENESS_CHECK_OPCODES(pOutgoing);
1789	RT_NOREF(pCallEntry);
1790	}
1791	#endif
1792
1793
1794	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1795	/**
1796	* Built-in function for checking the PC and checking opcodes after conditional
1797	* branching within the same page.
1798	*
1799	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1800	*/
1801	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1802	{
1803	PCIEMTB const pTb = pReNative->pTbOrg;
1804	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1805	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1806	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1807	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1808	BODY_SET_CUR_INSTR();
1809	BODY_FLUSH_PENDING_WRITES();
1810	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1811	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1812	//LogFunc(("okay\n"));
1813	return off;
1814	}
1815
1816	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1817	{
1818	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1819	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1820	LIVENESS_CHECK_OPCODES(pOutgoing);
1821	RT_NOREF(pCallEntry);
1822	}
1823	#endif
1824
1825
1826	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1827	/**
1828	* Built-in function for checking the PC and checking opcodes and considering
1829	* the need for CS.LIM checking after conditional branching within the same
1830	* page.
1831	*
1832	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1833	*/
1834	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1835	{
1836	PCIEMTB const pTb = pReNative->pTbOrg;
1837	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1838	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1839	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1840	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1841	BODY_SET_CUR_INSTR();
1842	BODY_FLUSH_PENDING_WRITES();
1843	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1844	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1845	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1846	//LogFunc(("okay\n"));
1847	return off;
1848	}
1849
1850	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1851	{
1852	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1853	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1854	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1855	LIVENESS_CHECK_OPCODES(pOutgoing);
1856	RT_NOREF(pCallEntry);
1857	}
1858	#endif
1859
1860
1861	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1862	/**
1863	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1864	* transitioning to a different code page.
1865	*
1866	* The code page transition can either be natural over onto the next page (with
1867	* the instruction starting at page offset zero) or by means of branching.
1868	*
1869	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
1870	*/
1871	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1872	{
1873	PCIEMTB const pTb = pReNative->pTbOrg;
1874	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1875	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1876	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1877	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1878	BODY_SET_CUR_INSTR();
1879	BODY_FLUSH_PENDING_WRITES();
1880	BODY_CHECK_CS_LIM(cbInstr);
1881	Assert(offRange == 0);
1882	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1883	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1884	//LogFunc(("okay\n"));
1885	return off;
1886	}
1887
1888	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1889	{
1890	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1891	LIVENESS_CHECK_CS_LIM(pOutgoing);
1892	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1893	LIVENESS_CHECK_OPCODES(pOutgoing);
1894	RT_NOREF(pCallEntry);
1895	}
1896	#endif
1897
1898
1899	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
1900	/**
1901	* Built-in function for loading TLB and checking opcodes when transitioning to
1902	* a different code page.
1903	*
1904	* The code page transition can either be natural over onto the next page (with
1905	* the instruction starting at page offset zero) or by means of branching.
1906	*
1907	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1908	*/
1909	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
1910	{
1911	PCIEMTB const pTb = pReNative->pTbOrg;
1912	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1913	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1914	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1915	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1916	BODY_SET_CUR_INSTR();
1917	BODY_FLUSH_PENDING_WRITES();
1918	Assert(offRange == 0);
1919	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1920	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1921	//LogFunc(("okay\n"));
1922	return off;
1923	}
1924
1925	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
1926	{
1927	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1928	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1929	LIVENESS_CHECK_OPCODES(pOutgoing);
1930	RT_NOREF(pCallEntry);
1931	}
1932	#endif
1933
1934
1935	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1936	/**
1937	* Built-in function for loading TLB and checking opcodes and considering the
1938	* need for CS.LIM checking when transitioning to a different code page.
1939	*
1940	* The code page transition can either be natural over onto the next page (with
1941	* the instruction starting at page offset zero) or by means of branching.
1942	*
1943	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1944	*/
1945	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1946	{
1947	PCIEMTB const pTb = pReNative->pTbOrg;
1948	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1949	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1950	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1951	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1952	BODY_SET_CUR_INSTR();
1953	BODY_FLUSH_PENDING_WRITES();
1954	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1955	Assert(offRange == 0);
1956	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1957	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1958	//LogFunc(("okay\n"));
1959	return off;
1960	}
1961
1962	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1963	{
1964	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1965	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1966	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1967	LIVENESS_CHECK_OPCODES(pOutgoing);
1968	RT_NOREF(pCallEntry);
1969	}
1970	#endif
1971
1972
1973
1974	/*
1975	* Natural page crossing checkers.
1976	*/
1977
1978	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
1979	/**
1980	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
1981	* both pages when transitioning to a different code page.
1982	*
1983	* This is used when the previous instruction requires revalidation of opcodes
1984	* bytes and the current instruction stries a page boundrary with opcode bytes
1985	* in both the old and new page.
1986	*
1987	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
1988	*/
1989	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1990	{
1991	PCIEMTB const pTb = pReNative->pTbOrg;
1992	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1993	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
1994	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
1995	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
1996	uint32_t const idxRange2 = idxRange1 + 1;
1997	BODY_SET_CUR_INSTR();
1998	BODY_FLUSH_PENDING_WRITES();
1999	BODY_CHECK_CS_LIM(cbInstr);
2000	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2001	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2002	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2003	return off;
2004	}
2005
2006	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2007	{
2008	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2009	LIVENESS_CHECK_CS_LIM(pOutgoing);
2010	LIVENESS_CHECK_OPCODES(pOutgoing);
2011	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2012	RT_NOREF(pCallEntry);
2013	}
2014	#endif
2015
2016
2017	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2018	/**
2019	* Built-in function for loading TLB and checking opcodes on both pages when
2020	* transitioning to a different code page.
2021	*
2022	* This is used when the previous instruction requires revalidation of opcodes
2023	* bytes and the current instruction stries a page boundrary with opcode bytes
2024	* in both the old and new page.
2025	*
2026	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2027	*/
2028	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2029	{
2030	PCIEMTB const pTb = pReNative->pTbOrg;
2031	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2032	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2033	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2034	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2035	uint32_t const idxRange2 = idxRange1 + 1;
2036	BODY_SET_CUR_INSTR();
2037	BODY_FLUSH_PENDING_WRITES();
2038	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2039	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2040	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2041	return off;
2042	}
2043
2044	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2045	{
2046	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2047	LIVENESS_CHECK_OPCODES(pOutgoing);
2048	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2049	RT_NOREF(pCallEntry);
2050	}
2051	#endif
2052
2053
2054	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2055	/**
2056	* Built-in function for loading TLB and checking opcodes on both pages and
2057	* considering the need for CS.LIM checking when transitioning to a different
2058	* code page.
2059	*
2060	* This is used when the previous instruction requires revalidation of opcodes
2061	* bytes and the current instruction stries a page boundrary with opcode bytes
2062	* in both the old and new page.
2063	*
2064	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2065	*/
2066	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2067	{
2068	PCIEMTB const pTb = pReNative->pTbOrg;
2069	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2070	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2071	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2072	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2073	uint32_t const idxRange2 = idxRange1 + 1;
2074	BODY_SET_CUR_INSTR();
2075	BODY_FLUSH_PENDING_WRITES();
2076	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2077	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2078	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2079	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2080	return off;
2081	}
2082
2083	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2084	{
2085	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2086	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2087	LIVENESS_CHECK_OPCODES(pOutgoing);
2088	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2089	RT_NOREF(pCallEntry);
2090	}
2091	#endif
2092
2093
2094	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2095	/**
2096	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2097	* advancing naturally to a different code page.
2098	*
2099	* Only opcodes on the new page is checked.
2100	*
2101	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2102	*/
2103	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2104	{
2105	PCIEMTB const pTb = pReNative->pTbOrg;
2106	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2107	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2108	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2109	//uint32_t const offRange1 = (uint32_t)uParam2;
2110	uint32_t const idxRange2 = idxRange1 + 1;
2111	BODY_SET_CUR_INSTR();
2112	BODY_FLUSH_PENDING_WRITES();
2113	BODY_CHECK_CS_LIM(cbInstr);
2114	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2115	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2116	return off;
2117	}
2118
2119	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2120	{
2121	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2122	LIVENESS_CHECK_CS_LIM(pOutgoing);
2123	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2124	LIVENESS_CHECK_OPCODES(pOutgoing);
2125	RT_NOREF(pCallEntry);
2126	}
2127	#endif
2128
2129
2130	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2131	/**
2132	* Built-in function for loading TLB and checking opcodes when advancing
2133	* naturally to a different code page.
2134	*
2135	* Only opcodes on the new page is checked.
2136	*
2137	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2138	*/
2139	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2140	{
2141	PCIEMTB const pTb = pReNative->pTbOrg;
2142	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2143	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2144	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2145	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2146	uint32_t const idxRange2 = idxRange1 + 1;
2147	BODY_SET_CUR_INSTR();
2148	BODY_FLUSH_PENDING_WRITES();
2149	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2150	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2151	return off;
2152	}
2153
2154	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2155	{
2156	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2157	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2158	LIVENESS_CHECK_OPCODES(pOutgoing);
2159	RT_NOREF(pCallEntry);
2160	}
2161	#endif
2162
2163
2164	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2165	/**
2166	* Built-in function for loading TLB and checking opcodes and considering the
2167	* need for CS.LIM checking when advancing naturally to a different code page.
2168	*
2169	* Only opcodes on the new page is checked.
2170	*
2171	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2172	*/
2173	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2174	{
2175	PCIEMTB const pTb = pReNative->pTbOrg;
2176	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2177	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2178	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2179	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2180	uint32_t const idxRange2 = idxRange1 + 1;
2181	BODY_SET_CUR_INSTR();
2182	BODY_FLUSH_PENDING_WRITES();
2183	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2184	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2185	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2186	return off;
2187	}
2188
2189	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2190	{
2191	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2192	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2193	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2194	LIVENESS_CHECK_OPCODES(pOutgoing);
2195	RT_NOREF(pCallEntry);
2196	}
2197	#endif
2198
2199
2200	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2201	/**
2202	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2203	* advancing naturally to a different code page with first instr at byte 0.
2204	*
2205	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2206	*/
2207	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2208	{
2209	PCIEMTB const pTb = pReNative->pTbOrg;
2210	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2211	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2212	BODY_SET_CUR_INSTR();
2213	BODY_FLUSH_PENDING_WRITES();
2214	BODY_CHECK_CS_LIM(cbInstr);
2215	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2216	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2217	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2218	return off;
2219	}
2220
2221	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2222	{
2223	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2224	LIVENESS_CHECK_CS_LIM(pOutgoing);
2225	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2226	LIVENESS_CHECK_OPCODES(pOutgoing);
2227	RT_NOREF(pCallEntry);
2228	}
2229	#endif
2230
2231
2232	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2233	/**
2234	* Built-in function for loading TLB and checking opcodes when advancing
2235	* naturally to a different code page with first instr at byte 0.
2236	*
2237	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2238	*/
2239	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2240	{
2241	PCIEMTB const pTb = pReNative->pTbOrg;
2242	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2243	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2244	BODY_SET_CUR_INSTR();
2245	BODY_FLUSH_PENDING_WRITES();
2246	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2247	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2248	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2249	return off;
2250	}
2251
2252	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2253	{
2254	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2255	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2256	LIVENESS_CHECK_OPCODES(pOutgoing);
2257	RT_NOREF(pCallEntry);
2258	}
2259	#endif
2260
2261
2262	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2263	/**
2264	* Built-in function for loading TLB and checking opcodes and considering the
2265	* need for CS.LIM checking when advancing naturally to a different code page
2266	* with first instr at byte 0.
2267	*
2268	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2269	*/
2270	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2271	{
2272	PCIEMTB const pTb = pReNative->pTbOrg;
2273	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2274	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2275	BODY_SET_CUR_INSTR();
2276	BODY_FLUSH_PENDING_WRITES();
2277	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2278	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2279	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2280	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2281	return off;
2282	}
2283
2284	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2285	{
2286	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2287	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2288	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2289	LIVENESS_CHECK_OPCODES(pOutgoing);
2290	RT_NOREF(pCallEntry);
2291	}
2292	#endif
2293

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp@ 103778

Download in other formats: