IEMAllN8veRecompBltIn.cpp@ 105284

Last change on this file since 105284 was 105271, checked in by vboxsync, 7 months ago
VMM/IEM: Replaced IEMNATIVEEXITREASON with IEMNATIVELABELTYPE, since it's always been a super set of it. Some source code width adjustments. bugref:10677
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 95.7 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 105271 2024-07-11 10:30:56Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	#define IEM_WITH_OPAQUE_DECODER_STATE
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/cpum.h>
38	#include <VBox/vmm/dbgf.h>
39	#include "IEMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/log.h>
42	#include <VBox/err.h>
43	#include <VBox/param.h>
44	#include <iprt/assert.h>
45	#include <iprt/string.h>
46	#if defined(RT_ARCH_AMD64)
47	# include <iprt/x86.h>
48	#elif defined(RT_ARCH_ARM64)
49	# include <iprt/armv8.h>
50	#endif
51
52
53	#include "IEMInline.h"
54	#include "IEMThreadedFunctions.h"
55	#include "IEMN8veRecompiler.h"
56	#include "IEMN8veRecompilerEmit.h"
57	#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61	/*********************************************************************************************************************************
62	* TB Helper Functions *
63	*********************************************************************************************************************************/
64	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
65	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66	#endif
67
68
69	/**
70	* Used by TB code to deal with a TLB miss for a new page.
71	*/
72	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73	{
74	#ifdef IEM_WITH_TLB_STATISTICS
75	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
76	#endif
77	pVCpu->iem.s.pbInstrBuf = NULL;
78	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
79	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
80	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
81	if (pVCpu->iem.s.pbInstrBuf)
82	{ /* likely */ }
83	else
84	{
85	AssertMsgFailed(("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
86	IEM_DO_LONGJMP(pVCpu, VINF_SUCCESS);
87	}
88	}
89
90
91	/**
92	* Used by TB code to deal with a TLB miss for a new page.
93	*/
94	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
95	{
96	#ifdef IEM_WITH_TLB_STATISTICS
97	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
98	#endif
99	pVCpu->iem.s.pbInstrBuf = NULL;
100	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
101	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
102	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
103	AssertMsg(pVCpu->iem.s.pbInstrBuf, ("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
104	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
105	}
106
107
108	/*********************************************************************************************************************************
109	* Builtin functions *
110	*********************************************************************************************************************************/
111
112	/**
113	* Built-in function that does nothing.
114	*
115	* Whether this is called or not can be controlled by the entry in the
116	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
117	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
118	* whether it's the reduced call count in the TBs or the threaded calls flushing
119	* register state.
120	*/
121	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
122	{
123	RT_NOREF(pReNative, pCallEntry);
124	return off;
125	}
126
127	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
128	{
129	pOutgoing = pIncoming;
130	RT_NOREF(pCallEntry);
131	}
132
133
134	/**
135	* Emits for for LogCpuState.
136	*
137	* This shouldn't have any relevant impact on the recompiler state.
138	*/
139	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
140	{
141	#ifdef RT_ARCH_AMD64
142	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
143	/* push rax */
144	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
145	/* push imm32 */
146	pbCodeBuf[off++] = 0x68;
147	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
148	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
149	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
150	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
151	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
152	pbCodeBuf[off++] = X86_OP_REX_W;
153	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
154	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
155	off += sizeof(uint64_t);
156	/* call rax */
157	pbCodeBuf[off++] = 0xff;
158	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
159	/* pop rax */
160	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
161	/* pop rax */
162	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
163	#else
164	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
165	RT_NOREF(pCallEntry);
166	#endif
167
168	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
169	return off;
170	}
171
172	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
173	{
174	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
175	RT_NOREF(pCallEntry);
176	}
177
178
179	/**
180	* Built-in function that calls a C-implemention function taking zero arguments.
181	*/
182	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
183	{
184	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
185	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
186	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
187	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
188	}
189
190	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
191	{
192	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
193	RT_NOREF(pCallEntry);
194	}
195
196
197	/**
198	* Flushes pending writes in preparation of raising an exception or aborting the TB.
199	*/
200	#define BODY_FLUSH_PENDING_WRITES() \
201	off = iemNativeRegFlushPendingWrites(pReNative, off);
202
203
204	/**
205	* Built-in function that checks for pending interrupts that can be delivered or
206	* forced action flags.
207	*
208	* This triggers after the completion of an instruction, so EIP is already at
209	* the next instruction. If an IRQ or important FF is pending, this will return
210	* a non-zero status that stops TB execution.
211	*/
212	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
213	{
214	RT_NOREF(pCallEntry);
215
216	BODY_FLUSH_PENDING_WRITES();
217
218	/* It's too convenient to use iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet below
219	and I'm too lazy to create a 'Fixed' version of that one. */
220	uint32_t const idxLabelVmCheck = iemNativeLabelCreate(pReNative, kIemNativeLabelType_CheckIrq,
221	UINT32_MAX, pReNative->uCheckIrqSeqNo++);
222
223	/* Again, we need to load the extended EFLAGS before we actually need them
224	in case we jump. We couldn't use iemNativeRegAllocTmpForGuestReg if we
225	loaded them inside the check, as the shadow state would not be correct
226	when the code branches before the load. Ditto PC. */
227	uint8_t const idxEflReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_EFlags,
228	kIemNativeGstRegUse_ReadOnly);
229
230	uint8_t const idxPcReg = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc, kIemNativeGstRegUse_ReadOnly);
231
232	uint8_t idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
233
234	/*
235	* Start by checking the local forced actions of the EMT we're on for IRQs
236	* and other FFs that needs servicing.
237	*/
238	/** @todo this isn't even close to the NMI and interrupt conditions in EM! */
239	/* Load FFs in to idxTmpReg and AND with all relevant flags. */
240	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
241	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
242	VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
243	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
244	\| VMCPU_FF_TLB_FLUSH
245	\| VMCPU_FF_UNHALT ),
246	true /fSetFlags/);
247	/* If we end up with ZERO in idxTmpReg there is nothing to do.*/
248	uint32_t const offFixupJumpToVmCheck1 = off;
249	off = iemNativeEmitJzToFixed(pReNative, off, off /* ASSUME jz rel8 suffices */);
250
251	/* Some relevant FFs are set, but if's only APIC or/and PIC being set,
252	these may be supressed by EFLAGS.IF or CPUMIsInInterruptShadow. */
253	off = iemNativeEmitAndGprByImm(pReNative, off, idxTmpReg,
254	~(VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC), true /fSetFlags/);
255	/* Return VINF_IEM_REEXEC_BREAK if other FFs are set. */
256	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakFF);
257
258	/* So, it's only interrupt releated FFs and we need to see if IRQs are being
259	suppressed by the CPU or not. */
260	off = iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(pReNative, off, idxEflReg, X86_EFL_IF_BIT, idxLabelVmCheck);
261	off = iemNativeEmitTestAnyBitsInGprAndTbExitIfNoneSet(pReNative, off, idxEflReg, CPUMCTX_INHIBIT_SHADOW,
262	kIemNativeLabelType_ReturnBreakFF);
263
264	/* We've got shadow flags set, so we must check that the PC they are valid
265	for matches our current PC value. */
266	/** @todo AMD64 can do this more efficiently w/o loading uRipInhibitInt into
267	* a register. */
268	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.uRipInhibitInt));
269	off = iemNativeEmitTestIfGprNotEqualGprAndTbExit(pReNative, off, idxTmpReg, idxPcReg,
270	kIemNativeLabelType_ReturnBreakFF);
271
272	/*
273	* Now check the force flags of the VM.
274	*/
275	iemNativeLabelDefine(pReNative, idxLabelVmCheck, off);
276	iemNativeFixupFixedJump(pReNative, offFixupJumpToVmCheck1, off);
277	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, CTX_SUFF(pVM))); /* idxTmpReg = pVM */
278	off = iemNativeEmitLoadGprByGprU32(pReNative, off, idxTmpReg, idxTmpReg, RT_UOFFSETOF(VMCC, fGlobalForcedActions));
279	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, VM_FF_ALL_MASK, true /fSetFlags/);
280	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ReturnBreakFF);
281
282	/** @todo STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatCheckIrqBreaks); */
283
284	/*
285	* We're good, no IRQs or FFs pending.
286	*/
287	iemNativeRegFreeTmp(pReNative, idxTmpReg);
288	iemNativeRegFreeTmp(pReNative, idxEflReg);
289	iemNativeRegFreeTmp(pReNative, idxPcReg);
290
291	/*
292	* Note down that we've been here, so we can skip FFs + IRQ checks when
293	* doing direct linking.
294	*/
295	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
296	pReNative->idxLastCheckIrqCallNo = pReNative->idxCurCall;
297	#else
298	pReNative->idxLastCheckIrqCallNo = pCallEntry - pReNative->pTbOrg->Thrd.paCalls;
299	#endif
300
301	return off;
302	}
303
304	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
305	{
306	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
307	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
308	RT_NOREF(pCallEntry);
309	}
310
311
312	/**
313	* Built-in function checks if IEMCPU::fExec has the expected value.
314	*/
315	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
316	{
317	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
318	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
319
320	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
321	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
322	off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
323	kIemNativeLabelType_ReturnBreak);
324	iemNativeRegFreeTmp(pReNative, idxTmpReg);
325
326	/* Maintain the recompiler fExec state. */
327	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
328	return off;
329	}
330
331	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
332	{
333	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
334	RT_NOREF(pCallEntry);
335	}
336
337
338	/**
339	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
340	*/
341	/** @todo Optimize this, so we don't set the same value more than once. Just
342	* needs some tracking. */
343	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
344	# define BODY_SET_CUR_INSTR() \
345	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
346	#else
347	# define BODY_SET_CUR_INSTR() ((void)0)
348	#endif
349
350
351	/**
352	* Macro that emits the 16/32-bit CS.LIM check.
353	*/
354	#define BODY_CHECK_CS_LIM(a_cbInstr) \
355	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
356
357	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
358	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
359
360	DECL_FORCE_INLINE(uint32_t)
361	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
362	{
363	Assert(cbInstr > 0);
364	Assert(cbInstr < 16);
365	#ifdef VBOX_STRICT
366	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
367	#endif
368
369	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
370	Assert(pReNative->Core.offPc == 0);
371	#endif
372
373	/*
374	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
375	* a temporary register for calculating the last address of the instruction.
376	*
377	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
378	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
379	* that last updated EIP here checked it already, and that we're therefore
380	* safe in the 32-bit wrap-around scenario to only check that the last byte
381	* is within CS.LIM. In the case of instruction-by-instruction advancing
382	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
383	* must be using 4KB granularity and the previous instruction was fine.
384	*/
385	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
386	kIemNativeGstRegUse_ReadOnly);
387	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
388	kIemNativeGstRegUse_ReadOnly);
389	#ifdef RT_ARCH_AMD64
390	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
391	#elif defined(RT_ARCH_ARM64)
392	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
393	#else
394	# error "Port me"
395	#endif
396
397	if (cbInstr != 1)
398	{
399	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
400
401	/*
402	* 1. idxRegTmp = idxRegPc + cbInstr;
403	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
404	*/
405	#ifdef RT_ARCH_AMD64
406	/* 1. lea tmp32, [Pc + cbInstr - 1] */
407	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
408	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
409	pbCodeBuf[off++] = 0x8d;
410	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
411	if ((idxRegPc & 7) == X86_GREG_xSP)
412	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
413	pbCodeBuf[off++] = cbInstr - 1;
414
415	/* 2. cmp tmp32(r), CsLim(r/m). */
416	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
417	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
418	pbCodeBuf[off++] = 0x3b;
419	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
420
421	#elif defined(RT_ARCH_ARM64)
422	/* 1. add tmp32, Pc, #cbInstr-1 */
423	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
424	/* 2. cmp tmp32, CsLim */
425	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
426	false /f64Bit/, true /fSetFlags/);
427
428	#endif
429	iemNativeRegFreeTmp(pReNative, idxRegTmp);
430	}
431	else
432	{
433	/*
434	* Here we can skip step 1 and compare PC and CS.LIM directly.
435	*/
436	#ifdef RT_ARCH_AMD64
437	/* 2. cmp eip(r), CsLim(r/m). */
438	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
439	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
440	pbCodeBuf[off++] = 0x3b;
441	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
442
443	#elif defined(RT_ARCH_ARM64)
444	/* 2. cmp Pc, CsLim */
445	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
446	false /f64Bit/, true /fSetFlags/);
447
448	#endif
449	}
450
451	/* 3. Jump if greater. */
452	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
453
454	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
455	iemNativeRegFreeTmp(pReNative, idxRegPc);
456	return off;
457	}
458
459
460	/**
461	* Macro that considers whether we need CS.LIM checking after a branch or
462	* crossing over to a new page.
463	*/
464	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
465	RT_NOREF(a_cbInstr); \
466	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
467
468	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
469	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
470	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
471
472	DECL_FORCE_INLINE(uint32_t)
473	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
474	{
475	#ifdef VBOX_STRICT
476	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
477	#endif
478
479	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
480	Assert(pReNative->Core.offPc == 0);
481	#endif
482
483	/*
484	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
485	* exactly:
486	*
487	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
488	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
489	* return fRet;
490	* return fRet \| IEMTB_F_CS_LIM_CHECKS;
491	*
492	*
493	* We need EIP, CS.LIM and CS.BASE here.
494	*/
495
496	/* Calculate the offFromLim first: */
497	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
498	kIemNativeGstRegUse_ReadOnly);
499	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
500	kIemNativeGstRegUse_ReadOnly);
501	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
502
503	#ifdef RT_ARCH_ARM64
504	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
505	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
506	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
507	#else
508	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
509	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
510	#endif
511
512	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
513	iemNativeRegFreeTmp(pReNative, idxRegPc);
514
515	/* Calculate the threshold level (right side). */
516	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
517	kIemNativeGstRegUse_ReadOnly);
518	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
519
520	#ifdef RT_ARCH_ARM64
521	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
522	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
523	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
524	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
525	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
526	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
527	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
528
529	#else
530	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
531	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
532	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
533	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
534	#endif
535
536	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
537
538	/* Compare the two and jump out if we're too close to the limit. */
539	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
540	off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
541
542	iemNativeRegFreeTmp(pReNative, idxRegRight);
543	iemNativeRegFreeTmp(pReNative, idxRegLeft);
544	return off;
545	}
546
547
548
549	/**
550	* Macro that implements opcode (re-)checking.
551	*/
552	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
553	RT_NOREF(a_cbInstr); \
554	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
555
556	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
557
558	#if 0 /* debugging aid */
559	bool g_fBpOnObsoletion = false;
560	# define BP_ON_OBSOLETION g_fBpOnObsoletion
561	#else
562	# define BP_ON_OBSOLETION 0
563	#endif
564
565	DECL_FORCE_INLINE(uint32_t)
566	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
567	{
568	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
569	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
570	#ifdef VBOX_STRICT
571	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
572	#endif
573
574	/*
575	* Where to start and how much to compare.
576	*
577	* Looking at the ranges produced when r160746 was running a DOS VM with TB
578	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
579	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
580	*
581	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
582	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
583	*/
584	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
585	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
586	Assert(cbLeft > 0);
587	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
588	uint32_t offConsolidatedJump = UINT32_MAX;
589
590	#ifdef RT_ARCH_AMD64
591	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
592	completely inlined, for larger we use REPE CMPS. */
593	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
594	pbCodeBuf[off++] = a_bOpcode; \
595	Assert(offPage < 127); \
596	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
597	pbCodeBuf[off++] = RT_BYTE1(offPage); \
598	} while (0)
599
600	# ifdef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE
601	# define NEAR_JMP_SIZE 5
602	# else
603	# define NEAR_JMP_SIZE 6
604	# endif
605
606	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
607	if (offConsolidatedJump != UINT32_MAX) \
608	{ \
609	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
610	Assert(offDisp >= -128); \
611	pbCodeBuf[off++] = 0x75; /* jnz near */ \
612	pbCodeBuf[off++] = (uint8_t)offDisp; \
613	} \
614	else \
615	{ \
616	pbCodeBuf[off++] = 0x74; /* jz near +NEAR_JMP_SIZE */ \
617	pbCodeBuf[off++] = NEAR_JMP_SIZE + BP_ON_OBSOLETION; \
618	offConsolidatedJump = off; \
619	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
620	off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeLabelType_ObsoleteTb); \
621	} \
622	} while (0)
623
624	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
625	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
626	pbCodeBuf[off++] = *pbOpcodes++; \
627	pbCodeBuf[off++] = *pbOpcodes++; \
628	pbCodeBuf[off++] = *pbOpcodes++; \
629	pbCodeBuf[off++] = *pbOpcodes++; \
630	cbLeft -= 4; \
631	offPage += 4; \
632	CHECK_OPCODES_CMP_JMP(); \
633	} while (0)
634
635	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
636	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
637	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
638	pbCodeBuf[off++] = *pbOpcodes++; \
639	pbCodeBuf[off++] = *pbOpcodes++; \
640	cbLeft -= 2; \
641	offPage += 2; \
642	CHECK_OPCODES_CMP_JMP(); \
643	} while (0)
644
645	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
646	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
647	pbCodeBuf[off++] = *pbOpcodes++; \
648	cbLeft -= 1; \
649	offPage += 1; \
650	CHECK_OPCODES_CMP_JMP(); \
651	} while (0)
652
653	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
654	if (a_bPrefix) \
655	pbCodeBuf[off++] = (a_bPrefix); \
656	pbCodeBuf[off++] = (a_bOpcode); \
657	CHECK_OPCODES_CMP_JMP(); \
658	cbLeft -= (a_cbToSubtract); \
659	} while (0)
660
661	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
662	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
663	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
664	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
665	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
666	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
667	} while (0)
668
669	if (cbLeft <= 24)
670	{
671	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
672	( RT_BIT_32(X86_GREG_xAX)
673	\| RT_BIT_32(X86_GREG_xCX)
674	\| RT_BIT_32(X86_GREG_xDX)
675	\| RT_BIT_32(X86_GREG_xBX)
676	\| RT_BIT_32(X86_GREG_xSI)
677	\| RT_BIT_32(X86_GREG_xDI))
678	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
679	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
680	if (offPage >= 128 - cbLeft)
681	{
682	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
683	offPage &= 3;
684	}
685
686	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */);
687
688	if (cbLeft > 8)
689	switch (offPage & 3)
690	{
691	case 0:
692	break;
693	case 1: /* cost: 6 + 8 = 14 */
694	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
695	RT_FALL_THRU();
696	case 2: /* cost: 8 */
697	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
698	break;
699	case 3: /* cost: 6 */
700	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
701	break;
702	}
703
704	while (cbLeft >= 4)
705	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
706
707	if (cbLeft >= 2)
708	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
709	if (cbLeft)
710	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
711
712	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
713	iemNativeRegFreeTmp(pReNative, idxRegTmp);
714	}
715	else
716	{
717	/* RDI = &pbInstrBuf[offPage] */
718	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
719	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
720	if (offPage != 0)
721	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
722
723	/* RSI = pbOpcodes */
724	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
725	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
726
727	/* RCX = counts. */
728	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
729
730	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 36/);
731
732	/** @todo profile and optimize this further. Maybe an idea to align by
733	* offPage if the two cannot be reconsidled. */
734	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
735	switch (offPage & 7) /* max cost: 10 */
736	{
737	case 0:
738	break;
739	case 1: /* cost: 3+4+3 = 10 */
740	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
741	RT_FALL_THRU();
742	case 2: /* cost: 4+3 = 7 */
743	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
744	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
745	break;
746	case 3: /* cost: 3+3 = 6 */
747	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
748	RT_FALL_THRU();
749	case 4: /* cost: 3 */
750	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
751	break;
752	case 5: /* cost: 3+4 = 7 */
753	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
754	RT_FALL_THRU();
755	case 6: /* cost: 4 */
756	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
757	break;
758	case 7: /* cost: 3 */
759	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
760	break;
761	}
762
763	/* Compare qwords: */
764	uint32_t const cQWords = cbLeft >> 3;
765	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
766
767	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
768	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
769	cbLeft &= 7;
770
771	if (cbLeft & 4)
772	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
773	if (cbLeft & 2)
774	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
775	if (cbLeft & 1)
776	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
777
778	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
779	iemNativeRegFreeTmp(pReNative, idxRegCx);
780	iemNativeRegFreeTmp(pReNative, idxRegSi);
781	iemNativeRegFreeTmp(pReNative, idxRegDi);
782	}
783
784	#elif defined(RT_ARCH_ARM64)
785	/* We need pbInstrBuf in a register, whatever we do. */
786	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
787	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
788
789	/* We also need at least one more register for holding bytes & words we
790	load via pbInstrBuf. */
791	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
792
793	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
794
795	/* One byte compare can be done with the opcode byte as an immediate. We'll
796	do this to uint16_t align src1. */
797	bool fPendingJmp = RT_BOOL(offPage & 1);
798	if (fPendingJmp)
799	{
800	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
801	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
802	offPage += 1;
803	cbLeft -= 1;
804	}
805
806	if (cbLeft > 0)
807	{
808	/* We need a register for holding the opcode bytes we're comparing with,
809	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
810	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
811
812	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
813	if ((offPage & 3) && cbLeft >= 2)
814	{
815	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
816	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
817	if (fPendingJmp)
818	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
819	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
820	else
821	{
822	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
823	fPendingJmp = true;
824	}
825	pbOpcodes += 2;
826	offPage += 2;
827	cbLeft -= 2;
828	}
829
830	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
831	if ((offPage & 7) && cbLeft >= 4)
832	{
833	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
834	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
835	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
836	if (fPendingJmp)
837	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
838	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
839	else
840	{
841	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
842	fPendingJmp = true;
843	}
844	pbOpcodes += 4;
845	offPage += 4;
846	cbLeft -= 4;
847	}
848
849	/*
850	* If we've got 16 bytes or more left, switch to memcmp-style.
851	*/
852	if (cbLeft >= 16)
853	{
854	/* We need a pointer to the copy of the original opcode bytes. */
855	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
856	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
857
858	/* If there are more than 32 bytes to compare we create a loop, for
859	which we'll need a loop register. */
860	if (cbLeft >= 64)
861	{
862	if (fPendingJmp)
863	{
864	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
865	kArmv8InstrCond_Ne);
866	fPendingJmp = false;
867	}
868
869	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
870	uint16_t const cLoops = cbLeft / 32;
871	cbLeft = cbLeft % 32;
872	pbOpcodes += cLoops * 32;
873	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
874
875	if (offPage != 0) /** @todo optimize out this instruction. */
876	{
877	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
878	offPage = 0;
879	}
880
881	uint32_t const offLoopStart = off;
882	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
883	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
884	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
885
886	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
887	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
888	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
889	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
890
891	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
892	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
893	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
894	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
895
896	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
897	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
898	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
899	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
900
901	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
902	kArmv8InstrCond_Ne);
903
904	/* Advance and loop. */
905	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
906	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
907	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
908	pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
909	off++;
910
911	iemNativeRegFreeTmp(pReNative, idxRegLoop);
912	}
913
914	/* Deal with any remaining dwords (uint64_t). There can be up to
915	three if we looped and four if we didn't. */
916	uint32_t offSrc2 = 0;
917	while (cbLeft >= 8)
918	{
919	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
920	idxRegSrc1Ptr, offPage / 8);
921	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
922	idxRegSrc2Ptr, offSrc2 / 8);
923	if (fPendingJmp)
924	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
925	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
926	else
927	{
928	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
929	fPendingJmp = true;
930	}
931	pbOpcodes += 8;
932	offPage += 8;
933	offSrc2 += 8;
934	cbLeft -= 8;
935	}
936
937	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
938	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
939	}
940	/*
941	* Otherwise, we compare with constants and merge with the general mop-up.
942	*/
943	else
944	{
945	while (cbLeft >= 8)
946	{
947	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
948	offPage / 8);
949	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
950	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
951	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
952	if (fPendingJmp)
953	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
954	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
955	else
956	{
957	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
958	fPendingJmp = true;
959	}
960	pbOpcodes += 8;
961	offPage += 8;
962	cbLeft -= 8;
963	}
964	/* max cost thus far: 21 */
965	}
966
967	/* Deal with any remaining bytes (7 or less). */
968	Assert(cbLeft < 8);
969	if (cbLeft >= 4)
970	{
971	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
972	offPage / 4);
973	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
974	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
975	if (fPendingJmp)
976	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
977	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
978	else
979	{
980	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
981	fPendingJmp = true;
982	}
983	pbOpcodes += 4;
984	offPage += 4;
985	cbLeft -= 4;
986
987	}
988
989	if (cbLeft >= 2)
990	{
991	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
992	offPage / 2);
993	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
994	if (fPendingJmp)
995	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
996	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
997	else
998	{
999	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
1000	fPendingJmp = true;
1001	}
1002	pbOpcodes += 2;
1003	offPage += 2;
1004	cbLeft -= 2;
1005	}
1006
1007	if (cbLeft > 0)
1008	{
1009	Assert(cbLeft == 1);
1010	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
1011	if (fPendingJmp)
1012	{
1013	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
1014	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1015	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1016	}
1017	else
1018	{
1019	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
1020	fPendingJmp = true;
1021	}
1022	pbOpcodes += 1;
1023	offPage += 1;
1024	cbLeft -= 1;
1025	}
1026
1027	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1028	}
1029	Assert(cbLeft == 0);
1030
1031	/*
1032	* Finally, the branch on difference.
1033	*/
1034	if (fPendingJmp)
1035	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ObsoleteTb);
1036
1037	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
1038
1039	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1040	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1041	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1042	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1043
1044	#else
1045	# error "Port me"
1046	#endif
1047	return off;
1048	}
1049
1050
1051
1052	/**
1053	* Macro that implements PC check after a conditional branch.
1054	*/
1055	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1056	RT_NOREF(a_cbInstr); \
1057	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1058
1059	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1060	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1061	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1062	else do { } while (0)
1063
1064	DECL_FORCE_INLINE(uint32_t)
1065	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1066	uint8_t idxRange, uint16_t offRange)
1067	{
1068	#ifdef VBOX_STRICT
1069	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1070	#endif
1071
1072	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1073	Assert(pReNative->Core.offPc == 0);
1074	#endif
1075
1076	/*
1077	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1078	* constant for us here.
1079	*
1080	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1081	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1082	* it, and as long as we don't implement code TLB reload code here there is
1083	* no point in checking that the TLB data we're using is still valid.
1084	*
1085	* What we to do is.
1086	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1087	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1088	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1089	* we're in the wrong spot and need to find a new TB.
1090	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1091	* GCPhysRangePageWithOffset constant mentioned above.
1092	*
1093	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1094	* in 64-bit code or flat 32-bit.
1095	*/
1096
1097	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1098	the temp register, so we don't accidentally clobber something we'll be
1099	needing again immediately. This is why we get idxRegCsBase here. */
1100	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1101	kIemNativeGstRegUse_ReadOnly);
1102	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1103	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1104	kIemNativeGstRegUse_ReadOnly);
1105
1106	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1107
1108	#ifdef VBOX_STRICT
1109	/* Do assertions before idxRegTmp contains anything. */
1110	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1111	# ifdef RT_ARCH_AMD64
1112	{
1113	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1114	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1115	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1116	{
1117	/* cmp r/m64, imm8 */
1118	pbCodeBuf[off++] = X86_OP_REX_W;
1119	pbCodeBuf[off++] = 0x83;
1120	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1121	pbCodeBuf[off++] = 0;
1122	/* je rel8 */
1123	pbCodeBuf[off++] = 0x74;
1124	pbCodeBuf[off++] = 1;
1125	/* int3 */
1126	pbCodeBuf[off++] = 0xcc;
1127
1128	}
1129
1130	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1131	/* test r/m64, imm32 */
1132	pbCodeBuf[off++] = X86_OP_REX_W;
1133	pbCodeBuf[off++] = 0xf7;
1134	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1135	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1136	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1137	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1138	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1139	/* jz rel8 */
1140	pbCodeBuf[off++] = 0x74;
1141	pbCodeBuf[off++] = 1;
1142	/* int3 */
1143	pbCodeBuf[off++] = 0xcc;
1144	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1145	}
1146	# else
1147
1148	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1149	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1150	{
1151	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1152	# ifdef RT_ARCH_ARM64
1153	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1154	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1155	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1156	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1157	# else
1158	# error "Port me!"
1159	# endif
1160	}
1161	# endif
1162
1163	#endif /* VBOX_STRICT */
1164
1165	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1166	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1167	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1168	{
1169	#ifdef RT_ARCH_ARM64
1170	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1171	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1172	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1173	#else
1174	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1175	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1176	#endif
1177	}
1178	else
1179	{
1180	#ifdef RT_ARCH_ARM64
1181	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1182	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1183	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1184	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1185	#else
1186	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1187	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1188	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1189	#endif
1190	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1191	}
1192	iemNativeRegFreeTmp(pReNative, idxRegPc);
1193
1194	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1195	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1196	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1197
1198	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1199	#ifdef RT_ARCH_AMD64
1200	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1201	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1202	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1203	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1204	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1205
1206	#elif defined(RT_ARCH_ARM64)
1207	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1208
1209	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1210	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1211	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1212	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1213
1214	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1215	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1216	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1217	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1218	# endif
1219	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1220	#else
1221	# error "Port me"
1222	#endif
1223
1224	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1225	\| pTb->aRanges[idxRange].offPhysPage)
1226	+ offRange;
1227	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1228	kIemNativeLabelType_CheckBranchMiss);
1229
1230	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1231	return off;
1232	}
1233
1234
1235	/**
1236	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1237	* instruction crossing into a new page.
1238	*
1239	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1240	*/
1241	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1242	RT_NOREF(a_cbInstr); \
1243	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1244
1245	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1246	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1247	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1248	else do { } while (0)
1249
1250	DECL_FORCE_INLINE(uint32_t)
1251	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1252	{
1253	#ifdef VBOX_STRICT
1254	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1255	#endif
1256
1257	/*
1258	* Define labels and allocate the register for holding the GCPhys of the new page.
1259	*/
1260	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1261	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1262	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1263	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1264	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1265	: UINT32_MAX;
1266
1267	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1268
1269	/*
1270	* Jump to the TLB lookup code.
1271	*/
1272	if (!TlbState.fSkip)
1273	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1274
1275	/*
1276	* TlbMiss:
1277	*
1278	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1279	*/
1280	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1281
1282	/* Save variables in volatile registers. */
1283	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1284	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1285
1286	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1287	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1288
1289	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1290	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1291
1292	/* Done setting up parameters, make the call. */
1293	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1294
1295	/* Move the result to the right register. */
1296	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1297	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1298
1299	/* Restore variables and guest shadow registers to volatile registers. */
1300	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1301	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1302
1303	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1304	if (!TlbState.fSkip)
1305	{
1306	/* end of TlbMiss - Jump to the done label. */
1307	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1308	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1309
1310	/*
1311	* TlbLookup:
1312	*/
1313	off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1314	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1315	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1316	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1317
1318	# ifdef IEM_WITH_TLB_STATISTICS
1319	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1320	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1321	# endif
1322
1323	/*
1324	* TlbDone:
1325	*/
1326	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1327	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1328	}
1329	#else
1330	RT_NOREF(idxLabelTlbMiss);
1331	#endif
1332
1333	/*
1334	* Now check the physical address of the page matches the expected one.
1335	*/
1336	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1337	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1338	kIemNativeLabelType_ObsoleteTb);
1339
1340	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1341	return off;
1342	}
1343
1344
1345	/**
1346	* Macro that implements TLB loading and updating pbInstrBuf updating when
1347	* branching or when crossing a page on an instruction boundrary.
1348	*
1349	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1350	* it is an inter-page branch and also check the page offset.
1351	*
1352	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1353	*/
1354	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1355	RT_NOREF(a_cbInstr); \
1356	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1357
1358	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1359	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1360	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1361	else do { } while (0)
1362
1363	DECL_FORCE_INLINE(uint32_t)
1364	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1365	{
1366	#ifdef VBOX_STRICT
1367	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1368	#endif
1369
1370	BODY_FLUSH_PENDING_WRITES();
1371
1372	/*
1373	* Define labels and allocate the register for holding the GCPhys of the new page.
1374	*/
1375	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1376	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1377	\| pTb->aRanges[idxRange].offPhysPage;
1378
1379	/*
1380	*
1381	* First check if RIP is within the current code.
1382	*
1383	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1384	* difference is what we do when stuff doesn't match up.
1385	*
1386	* What we to do is.
1387	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1388	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1389	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1390	* we need to retranslate RIP via the TLB.
1391	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1392	* GCPhysRangePageWithOffset constant mentioned above.
1393	*
1394	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1395	* in 64-bit code or flat 32-bit.
1396	*
1397	*/
1398
1399	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1400	the temp register, so we don't accidentally clobber something we'll be
1401	needing again immediately. This is why we get idxRegCsBase here.
1402	Update: We share registers with the TlbState, as the TLB code path has
1403	little in common with the rest of the code. */
1404	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1405	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1406	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1407	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1408	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1409	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1410	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1411	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1412
1413	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1414	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1415	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1416
1417	#ifdef VBOX_STRICT
1418	/* Do assertions before idxRegTmp contains anything. */
1419	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1420	# ifdef RT_ARCH_AMD64
1421	{
1422	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1423	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1424	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1425	{
1426	/* cmp r/m64, imm8 */
1427	pbCodeBuf[off++] = X86_OP_REX_W;
1428	pbCodeBuf[off++] = 0x83;
1429	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1430	pbCodeBuf[off++] = 0;
1431	/* je rel8 */
1432	pbCodeBuf[off++] = 0x74;
1433	pbCodeBuf[off++] = 1;
1434	/* int3 */
1435	pbCodeBuf[off++] = 0xcc;
1436
1437	}
1438
1439	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1440	/* test r/m64, imm32 */
1441	pbCodeBuf[off++] = X86_OP_REX_W;
1442	pbCodeBuf[off++] = 0xf7;
1443	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1444	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1445	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1446	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1447	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1448	/* jz rel8 */
1449	pbCodeBuf[off++] = 0x74;
1450	pbCodeBuf[off++] = 1;
1451	/* int3 */
1452	pbCodeBuf[off++] = 0xcc;
1453	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1454	}
1455	# else
1456
1457	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1458	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1459	{
1460	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1461	# ifdef RT_ARCH_ARM64
1462	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1463	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1464	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1465	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1466	# else
1467	# error "Port me!"
1468	# endif
1469	}
1470	# endif
1471
1472	#endif /* VBOX_STRICT */
1473
1474	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1475	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1476	2nd register to check if we've looped more than once already.*/
1477	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1478
1479	uint32_t const offLabelRedoChecks = off;
1480
1481	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1482	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1483	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1484	{
1485	#ifdef RT_ARCH_ARM64
1486	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1487	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1488	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1489	#else
1490	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1491	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1492	#endif
1493	}
1494	else
1495	{
1496	#ifdef RT_ARCH_ARM64
1497	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1498	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1499	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1500	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1501	#else
1502	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1503	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1504	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1505	#endif
1506	}
1507
1508	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1509	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1510	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1511	uint32_t const offFixedJumpToTlbLoad = off;
1512	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1513
1514	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1515	#ifdef RT_ARCH_AMD64
1516	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1517	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1518	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1519	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1520	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1521
1522	#elif defined(RT_ARCH_ARM64)
1523
1524	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1525	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1526	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1527	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1528
1529	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1530	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1531	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1532	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1533	# endif
1534	#else
1535	# error "Port me"
1536	#endif
1537
1538	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1539
1540	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1541	careful and avoid implicit temporary register usage here.
1542
1543	Unlike the threaded version of this code, we do not obsolete TBs here to
1544	reduce the code size and because indirect calls may legally end at the
1545	same offset in two different pages depending on the program state. */
1546	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1547	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1548	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1549	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1550	uint32_t const offFixedJumpToEnd = off;
1551	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1552
1553	/*
1554	* TlbLoad:
1555	*
1556	* First we try to go via the TLB.
1557	*/
1558	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1559
1560	/* Check that we haven't been here before. */
1561	off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2, false /f64Bit/,
1562	kIemNativeLabelType_CheckBranchMiss);
1563
1564	/* Jump to the TLB lookup code. */
1565	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1566	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1567	: UINT32_MAX;
1568	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1569	if (!TlbState.fSkip)
1570	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1571
1572	/*
1573	* TlbMiss:
1574	*
1575	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1576	*/
1577	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1578	RT_NOREF(idxLabelTlbMiss);
1579
1580	/* Save variables in volatile registers. */
1581	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1582	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1583	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1584
1585	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1586	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1587
1588	/* Done setting up parameters, make the call. */
1589	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1590
1591	/* Restore variables and guest shadow registers to volatile registers. */
1592	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1593	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1594	TlbState.getActiveRegsWithShadows()
1595	\| RT_BIT_32(idxRegPc)
1596	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1597
1598	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1599	if (!TlbState.fSkip)
1600	{
1601	/* end of TlbMiss - Jump to the done label. */
1602	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1603	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1604
1605	/*
1606	* TlbLookup:
1607	*/
1608	off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1609	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1610	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1611
1612	# ifdef IEM_WITH_TLB_STATISTICS
1613	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1614	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1615	# endif
1616
1617	/*
1618	* TlbDone:
1619	*/
1620	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1621	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1622	}
1623	#else
1624	RT_NOREF(idxLabelTlbMiss);
1625	#endif
1626
1627	/* Jmp back to the start and redo the checks. */
1628	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1629	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1630
1631	/*
1632	* End:
1633	*
1634	* The end.
1635	*/
1636	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1637
1638	if (!TlbState.fSkip)
1639	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1640	else
1641	{
1642	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1643	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1644	iemNativeRegFreeTmp(pReNative, idxRegPc);
1645	if (idxRegCsBase != UINT8_MAX)
1646	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1647	}
1648	return off;
1649	}
1650
1651
1652	#ifdef BODY_CHECK_CS_LIM
1653	/**
1654	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1655	* raising a \#GP(0) if this isn't the case.
1656	*/
1657	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1658	{
1659	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1660	BODY_SET_CUR_INSTR();
1661	BODY_FLUSH_PENDING_WRITES();
1662	BODY_CHECK_CS_LIM(cbInstr);
1663	return off;
1664	}
1665
1666	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1667	{
1668	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1669	LIVENESS_CHECK_CS_LIM(pOutgoing);
1670	RT_NOREF(pCallEntry);
1671	}
1672	#endif
1673
1674
1675	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1676	/**
1677	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1678	* that may have modified them.
1679	*/
1680	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1681	{
1682	PCIEMTB const pTb = pReNative->pTbOrg;
1683	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1684	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1685	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1686	BODY_SET_CUR_INSTR();
1687	BODY_FLUSH_PENDING_WRITES();
1688	BODY_CHECK_CS_LIM(cbInstr);
1689	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1690	return off;
1691	}
1692
1693	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1694	{
1695	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1696	LIVENESS_CHECK_CS_LIM(pOutgoing);
1697	LIVENESS_CHECK_OPCODES(pOutgoing);
1698	RT_NOREF(pCallEntry);
1699	}
1700	#endif
1701
1702
1703	#if defined(BODY_CHECK_OPCODES)
1704	/**
1705	* Built-in function for re-checking opcodes after an instruction that may have
1706	* modified them.
1707	*/
1708	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1709	{
1710	PCIEMTB const pTb = pReNative->pTbOrg;
1711	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1712	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1713	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1714	BODY_SET_CUR_INSTR();
1715	BODY_FLUSH_PENDING_WRITES();
1716	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1717	return off;
1718	}
1719
1720	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1721	{
1722	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1723	LIVENESS_CHECK_OPCODES(pOutgoing);
1724	RT_NOREF(pCallEntry);
1725	}
1726	#endif
1727
1728
1729	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1730	/**
1731	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1732	* checking after an instruction that may have modified them.
1733	*/
1734	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1735	{
1736	PCIEMTB const pTb = pReNative->pTbOrg;
1737	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1738	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1739	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1740	BODY_SET_CUR_INSTR();
1741	BODY_FLUSH_PENDING_WRITES();
1742	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1743	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1744	return off;
1745	}
1746
1747	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1748	{
1749	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1750	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1751	LIVENESS_CHECK_OPCODES(pOutgoing);
1752	RT_NOREF(pCallEntry);
1753	}
1754	#endif
1755
1756
1757	/*
1758	* Post-branching checkers.
1759	*/
1760
1761	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1762	/**
1763	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1764	* after conditional branching within the same page.
1765	*
1766	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1767	*/
1768	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1769	{
1770	PCIEMTB const pTb = pReNative->pTbOrg;
1771	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1772	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1773	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1774	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1775	BODY_SET_CUR_INSTR();
1776	BODY_FLUSH_PENDING_WRITES();
1777	BODY_CHECK_CS_LIM(cbInstr);
1778	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1779	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1780	//LogFunc(("okay\n"));
1781	return off;
1782	}
1783
1784	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1785	{
1786	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1787	LIVENESS_CHECK_CS_LIM(pOutgoing);
1788	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1789	LIVENESS_CHECK_OPCODES(pOutgoing);
1790	RT_NOREF(pCallEntry);
1791	}
1792	#endif
1793
1794
1795	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1796	/**
1797	* Built-in function for checking the PC and checking opcodes after conditional
1798	* branching within the same page.
1799	*
1800	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1801	*/
1802	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1803	{
1804	PCIEMTB const pTb = pReNative->pTbOrg;
1805	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1806	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1807	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1808	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1809	BODY_SET_CUR_INSTR();
1810	BODY_FLUSH_PENDING_WRITES();
1811	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1812	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1813	//LogFunc(("okay\n"));
1814	return off;
1815	}
1816
1817	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1818	{
1819	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1820	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1821	LIVENESS_CHECK_OPCODES(pOutgoing);
1822	RT_NOREF(pCallEntry);
1823	}
1824	#endif
1825
1826
1827	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1828	/**
1829	* Built-in function for checking the PC and checking opcodes and considering
1830	* the need for CS.LIM checking after conditional branching within the same
1831	* page.
1832	*
1833	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1834	*/
1835	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1836	{
1837	PCIEMTB const pTb = pReNative->pTbOrg;
1838	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1839	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1840	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1841	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1842	BODY_SET_CUR_INSTR();
1843	BODY_FLUSH_PENDING_WRITES();
1844	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1845	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1846	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1847	//LogFunc(("okay\n"));
1848	return off;
1849	}
1850
1851	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1852	{
1853	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1854	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1855	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1856	LIVENESS_CHECK_OPCODES(pOutgoing);
1857	RT_NOREF(pCallEntry);
1858	}
1859	#endif
1860
1861
1862	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1863	/**
1864	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1865	* transitioning to a different code page.
1866	*
1867	* The code page transition can either be natural over onto the next page (with
1868	* the instruction starting at page offset zero) or by means of branching.
1869	*
1870	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
1871	*/
1872	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1873	{
1874	PCIEMTB const pTb = pReNative->pTbOrg;
1875	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1876	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1877	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1878	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1879	BODY_SET_CUR_INSTR();
1880	BODY_FLUSH_PENDING_WRITES();
1881	BODY_CHECK_CS_LIM(cbInstr);
1882	Assert(offRange == 0);
1883	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1884	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1885	//LogFunc(("okay\n"));
1886	return off;
1887	}
1888
1889	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
1890	{
1891	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1892	LIVENESS_CHECK_CS_LIM(pOutgoing);
1893	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1894	LIVENESS_CHECK_OPCODES(pOutgoing);
1895	RT_NOREF(pCallEntry);
1896	}
1897	#endif
1898
1899
1900	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
1901	/**
1902	* Built-in function for loading TLB and checking opcodes when transitioning to
1903	* a different code page.
1904	*
1905	* The code page transition can either be natural over onto the next page (with
1906	* the instruction starting at page offset zero) or by means of branching.
1907	*
1908	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1909	*/
1910	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
1911	{
1912	PCIEMTB const pTb = pReNative->pTbOrg;
1913	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1914	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1915	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1916	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1917	BODY_SET_CUR_INSTR();
1918	BODY_FLUSH_PENDING_WRITES();
1919	Assert(offRange == 0);
1920	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1921	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1922	//LogFunc(("okay\n"));
1923	return off;
1924	}
1925
1926	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
1927	{
1928	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1929	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1930	LIVENESS_CHECK_OPCODES(pOutgoing);
1931	RT_NOREF(pCallEntry);
1932	}
1933	#endif
1934
1935
1936	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1937	/**
1938	* Built-in function for loading TLB and checking opcodes and considering the
1939	* need for CS.LIM checking when transitioning to a different code page.
1940	*
1941	* The code page transition can either be natural over onto the next page (with
1942	* the instruction starting at page offset zero) or by means of branching.
1943	*
1944	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
1945	*/
1946	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1947	{
1948	PCIEMTB const pTb = pReNative->pTbOrg;
1949	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1950	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1951	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1952	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1953	BODY_SET_CUR_INSTR();
1954	BODY_FLUSH_PENDING_WRITES();
1955	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1956	Assert(offRange == 0);
1957	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
1958	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1959	//LogFunc(("okay\n"));
1960	return off;
1961	}
1962
1963	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
1964	{
1965	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
1966	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1967	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
1968	LIVENESS_CHECK_OPCODES(pOutgoing);
1969	RT_NOREF(pCallEntry);
1970	}
1971	#endif
1972
1973
1974
1975	/*
1976	* Natural page crossing checkers.
1977	*/
1978
1979	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
1980	/**
1981	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
1982	* both pages when transitioning to a different code page.
1983	*
1984	* This is used when the previous instruction requires revalidation of opcodes
1985	* bytes and the current instruction stries a page boundrary with opcode bytes
1986	* in both the old and new page.
1987	*
1988	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
1989	*/
1990	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
1991	{
1992	PCIEMTB const pTb = pReNative->pTbOrg;
1993	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1994	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
1995	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
1996	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
1997	uint32_t const idxRange2 = idxRange1 + 1;
1998	BODY_SET_CUR_INSTR();
1999	BODY_FLUSH_PENDING_WRITES();
2000	BODY_CHECK_CS_LIM(cbInstr);
2001	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2002	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2003	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2004	return off;
2005	}
2006
2007	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2008	{
2009	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2010	LIVENESS_CHECK_CS_LIM(pOutgoing);
2011	LIVENESS_CHECK_OPCODES(pOutgoing);
2012	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2013	RT_NOREF(pCallEntry);
2014	}
2015	#endif
2016
2017
2018	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2019	/**
2020	* Built-in function for loading TLB and checking opcodes on both pages when
2021	* transitioning to a different code page.
2022	*
2023	* This is used when the previous instruction requires revalidation of opcodes
2024	* bytes and the current instruction stries a page boundrary with opcode bytes
2025	* in both the old and new page.
2026	*
2027	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2028	*/
2029	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2030	{
2031	PCIEMTB const pTb = pReNative->pTbOrg;
2032	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2033	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2034	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2035	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2036	uint32_t const idxRange2 = idxRange1 + 1;
2037	BODY_SET_CUR_INSTR();
2038	BODY_FLUSH_PENDING_WRITES();
2039	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2040	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2041	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2042	return off;
2043	}
2044
2045	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2046	{
2047	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2048	LIVENESS_CHECK_OPCODES(pOutgoing);
2049	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2050	RT_NOREF(pCallEntry);
2051	}
2052	#endif
2053
2054
2055	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2056	/**
2057	* Built-in function for loading TLB and checking opcodes on both pages and
2058	* considering the need for CS.LIM checking when transitioning to a different
2059	* code page.
2060	*
2061	* This is used when the previous instruction requires revalidation of opcodes
2062	* bytes and the current instruction stries a page boundrary with opcode bytes
2063	* in both the old and new page.
2064	*
2065	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2066	*/
2067	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2068	{
2069	PCIEMTB const pTb = pReNative->pTbOrg;
2070	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2071	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2072	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2073	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2074	uint32_t const idxRange2 = idxRange1 + 1;
2075	BODY_SET_CUR_INSTR();
2076	BODY_FLUSH_PENDING_WRITES();
2077	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2078	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2079	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2080	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2081	return off;
2082	}
2083
2084	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2085	{
2086	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2087	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2088	LIVENESS_CHECK_OPCODES(pOutgoing);
2089	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2090	RT_NOREF(pCallEntry);
2091	}
2092	#endif
2093
2094
2095	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2096	/**
2097	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2098	* advancing naturally to a different code page.
2099	*
2100	* Only opcodes on the new page is checked.
2101	*
2102	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2103	*/
2104	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2105	{
2106	PCIEMTB const pTb = pReNative->pTbOrg;
2107	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2108	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2109	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2110	//uint32_t const offRange1 = (uint32_t)uParam2;
2111	uint32_t const idxRange2 = idxRange1 + 1;
2112	BODY_SET_CUR_INSTR();
2113	BODY_FLUSH_PENDING_WRITES();
2114	BODY_CHECK_CS_LIM(cbInstr);
2115	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2116	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2117	return off;
2118	}
2119
2120	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2121	{
2122	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2123	LIVENESS_CHECK_CS_LIM(pOutgoing);
2124	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2125	LIVENESS_CHECK_OPCODES(pOutgoing);
2126	RT_NOREF(pCallEntry);
2127	}
2128	#endif
2129
2130
2131	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2132	/**
2133	* Built-in function for loading TLB and checking opcodes when advancing
2134	* naturally to a different code page.
2135	*
2136	* Only opcodes on the new page is checked.
2137	*
2138	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2139	*/
2140	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2141	{
2142	PCIEMTB const pTb = pReNative->pTbOrg;
2143	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2144	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2145	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2146	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2147	uint32_t const idxRange2 = idxRange1 + 1;
2148	BODY_SET_CUR_INSTR();
2149	BODY_FLUSH_PENDING_WRITES();
2150	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2151	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2152	return off;
2153	}
2154
2155	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2156	{
2157	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2158	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2159	LIVENESS_CHECK_OPCODES(pOutgoing);
2160	RT_NOREF(pCallEntry);
2161	}
2162	#endif
2163
2164
2165	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2166	/**
2167	* Built-in function for loading TLB and checking opcodes and considering the
2168	* need for CS.LIM checking when advancing naturally to a different code page.
2169	*
2170	* Only opcodes on the new page is checked.
2171	*
2172	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2173	*/
2174	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2175	{
2176	PCIEMTB const pTb = pReNative->pTbOrg;
2177	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2178	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2179	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2180	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2181	uint32_t const idxRange2 = idxRange1 + 1;
2182	BODY_SET_CUR_INSTR();
2183	BODY_FLUSH_PENDING_WRITES();
2184	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2185	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2186	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2187	return off;
2188	}
2189
2190	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2191	{
2192	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2193	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2194	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2195	LIVENESS_CHECK_OPCODES(pOutgoing);
2196	RT_NOREF(pCallEntry);
2197	}
2198	#endif
2199
2200
2201	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2202	/**
2203	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2204	* advancing naturally to a different code page with first instr at byte 0.
2205	*
2206	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2207	*/
2208	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2209	{
2210	PCIEMTB const pTb = pReNative->pTbOrg;
2211	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2212	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2213	BODY_SET_CUR_INSTR();
2214	BODY_FLUSH_PENDING_WRITES();
2215	BODY_CHECK_CS_LIM(cbInstr);
2216	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2217	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2218	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2219	return off;
2220	}
2221
2222	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2223	{
2224	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2225	LIVENESS_CHECK_CS_LIM(pOutgoing);
2226	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2227	LIVENESS_CHECK_OPCODES(pOutgoing);
2228	RT_NOREF(pCallEntry);
2229	}
2230	#endif
2231
2232
2233	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2234	/**
2235	* Built-in function for loading TLB and checking opcodes when advancing
2236	* naturally to a different code page with first instr at byte 0.
2237	*
2238	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2239	*/
2240	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2241	{
2242	PCIEMTB const pTb = pReNative->pTbOrg;
2243	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2244	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2245	BODY_SET_CUR_INSTR();
2246	BODY_FLUSH_PENDING_WRITES();
2247	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2248	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2249	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2250	return off;
2251	}
2252
2253	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2254	{
2255	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2256	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2257	LIVENESS_CHECK_OPCODES(pOutgoing);
2258	RT_NOREF(pCallEntry);
2259	}
2260	#endif
2261
2262
2263	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2264	/**
2265	* Built-in function for loading TLB and checking opcodes and considering the
2266	* need for CS.LIM checking when advancing naturally to a different code page
2267	* with first instr at byte 0.
2268	*
2269	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2270	*/
2271	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2272	{
2273	PCIEMTB const pTb = pReNative->pTbOrg;
2274	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2275	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2276	BODY_SET_CUR_INSTR();
2277	BODY_FLUSH_PENDING_WRITES();
2278	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2279	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2280	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2281	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2282	return off;
2283	}
2284
2285	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2286	{
2287	IEM_LIVENESS_RAW_INIT_WITH_XCPT_OR_CALL(pOutgoing, pIncoming);
2288	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2289	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2290	LIVENESS_CHECK_OPCODES(pOutgoing);
2291	RT_NOREF(pCallEntry);
2292	}
2293	#endif
2294

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp@ 105284

Download in other formats: