IEMAllN8veRecompBltIn.cpp@ 106125

Last change on this file since 106125 was 106125, checked in by vboxsync, 7 months ago
VMM/IEM: Eliminated the #ifndef IEMNATIVE_WITH_RECOMPILER_PER_CHUNK_TAIL_CODE bits to try reduce the #ifdef-spaghetti a little. bugref:10677
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 102.9 KB

Line
1	/* $Id: IEMAllN8veRecompBltIn.cpp 106125 2024-09-23 22:42:27Z vboxsync $ */
2	/** @file
3	* IEM - Native Recompiler, Emitters for Built-In Threaded Functions.
4	*/
5
6	/*
7	* Copyright (C) 2023-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_IEM_RE_NATIVE
33	#define IEM_WITH_OPAQUE_DECODER_STATE
34	#define VMCPU_INCL_CPUM_GST_CTX
35	#define VMM_INCLUDED_SRC_include_IEMMc_h /* block IEMMc.h inclusion. */
36	#include <VBox/vmm/iem.h>
37	#include <VBox/vmm/cpum.h>
38	#include <VBox/vmm/dbgf.h>
39	#include "IEMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include <VBox/log.h>
42	#include <VBox/err.h>
43	#include <VBox/param.h>
44	#include <iprt/assert.h>
45	#include <iprt/string.h>
46	#if defined(RT_ARCH_AMD64)
47	# include <iprt/x86.h>
48	#elif defined(RT_ARCH_ARM64)
49	# include <iprt/armv8.h>
50	#endif
51
52
53	#include "IEMInline.h"
54	#include "IEMThreadedFunctions.h"
55	#include "IEMN8veRecompiler.h"
56	#include "IEMN8veRecompilerEmit.h"
57	#include "IEMN8veRecompilerTlbLookup.h"
58
59
60
61	/*********************************************************************************************************************************
62	* TB Helper Functions *
63	*********************************************************************************************************************************/
64	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
65	DECLASM(void) iemNativeHlpAsmSafeWrapLogCpuState(void);
66	#endif
67
68
69	/**
70	* Used by TB code to deal with a TLB miss for a new page.
71	*/
72	IEM_DECL_NATIVE_HLP_DEF(void, iemNativeHlpMemCodeNewPageTlbMiss,(PVMCPUCC pVCpu))
73	{
74	#ifdef IEM_WITH_TLB_STATISTICS
75	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPage);
76	#endif
77	pVCpu->iem.s.pbInstrBuf = NULL;
78	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE;
79	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
80	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
81	if (pVCpu->iem.s.pbInstrBuf)
82	{ /* likely */ }
83	else
84	{
85	AssertMsgFailed(("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
86	IEM_DO_LONGJMP(pVCpu, VINF_SUCCESS);
87	}
88	}
89
90
91	/**
92	* Used by TB code to deal with a TLB miss for a new page.
93	*/
94	IEM_DECL_NATIVE_HLP_DEF(RTGCPHYS, iemNativeHlpMemCodeNewPageTlbMissWithOff,(PVMCPUCC pVCpu, uint8_t offInstr))
95	{
96	#ifdef IEM_WITH_TLB_STATISTICS
97	STAM_REL_COUNTER_INC(&pVCpu->iem.s.StatNativeCodeTlbMissesNewPageWithOffset);
98	#endif
99	pVCpu->iem.s.pbInstrBuf = NULL;
100	pVCpu->iem.s.offCurInstrStart = GUEST_PAGE_SIZE - offInstr;
101	pVCpu->iem.s.offInstrNextByte = GUEST_PAGE_SIZE;
102	iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
103	AssertMsg(pVCpu->iem.s.pbInstrBuf, ("cs:rip=%04x:%08RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
104	return pVCpu->iem.s.pbInstrBuf ? pVCpu->iem.s.GCPhysInstrBuf : NIL_RTGCPHYS;
105	}
106
107
108	/*********************************************************************************************************************************
109	* Builtin functions *
110	*********************************************************************************************************************************/
111
112	/**
113	* Built-in function that does nothing.
114	*
115	* Whether this is called or not can be controlled by the entry in the
116	* IEMThreadedGenerator.katBltIns table. This can be useful to determine
117	* whether why behaviour changes when enabling the LogCpuState builtins. I.e.
118	* whether it's the reduced call count in the TBs or the threaded calls flushing
119	* register state.
120	*/
121	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Nop)
122	{
123	RT_NOREF(pReNative, pCallEntry);
124	return off;
125	}
126
127	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Nop)
128	{
129	pOutgoing = pIncoming;
130	RT_NOREF(pCallEntry);
131	}
132
133
134	/**
135	* Emits for for LogCpuState.
136	*
137	* This shouldn't have any relevant impact on the recompiler state.
138	*/
139	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_LogCpuState)
140	{
141	#ifdef RT_ARCH_AMD64
142	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 20);
143	/* push rax */
144	pbCodeBuf[off++] = 0x50 + X86_GREG_xAX;
145	/* push imm32 */
146	pbCodeBuf[off++] = 0x68;
147	pbCodeBuf[off++] = RT_BYTE1(pCallEntry->auParams[0]);
148	pbCodeBuf[off++] = RT_BYTE2(pCallEntry->auParams[0]);
149	pbCodeBuf[off++] = RT_BYTE3(pCallEntry->auParams[0]);
150	pbCodeBuf[off++] = RT_BYTE4(pCallEntry->auParams[0]);
151	/* mov rax, iemNativeHlpAsmSafeWrapLogCpuState */
152	pbCodeBuf[off++] = X86_OP_REX_W;
153	pbCodeBuf[off++] = 0xb8 + X86_GREG_xAX;
154	(uint64_t )&pbCodeBuf[off] = (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState;
155	off += sizeof(uint64_t);
156	/* call rax */
157	pbCodeBuf[off++] = 0xff;
158	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);
159	/* pop rax */
160	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
161	/* pop rax */
162	pbCodeBuf[off++] = 0x58 + X86_GREG_xAX;
163	#else
164	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpAsmSafeWrapLogCpuState);
165	RT_NOREF(pCallEntry);
166	#endif
167
168	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
169	return off;
170	}
171
172	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_LogCpuState)
173	{
174	IEM_LIVENESS_RAW_INIT_WITH_CALL(pOutgoing, pIncoming);
175	RT_NOREF(pCallEntry);
176	}
177
178
179	/**
180	* Built-in function that calls a C-implemention function taking zero arguments.
181	*/
182	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_DeferToCImpl0)
183	{
184	PFNIEMCIMPL0 const pfnCImpl = (PFNIEMCIMPL0)(uintptr_t)pCallEntry->auParams[0];
185	uint8_t const cbInstr = (uint8_t)pCallEntry->auParams[1];
186	uint64_t const fGstShwFlush = pCallEntry->auParams[2];
187	return iemNativeEmitCImplCall(pReNative, off, pCallEntry->idxInstr, fGstShwFlush, (uintptr_t)pfnCImpl, cbInstr, 0, 0, 0, 0);
188	}
189
190	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_DeferToCImpl0)
191	{
192	IEM_LIVENESS_RAW_INIT_WITH_CALL(pOutgoing, pIncoming);
193	RT_NOREF(pCallEntry);
194	}
195
196
197	/**
198	* Flushes pending writes in preparation of raising an exception or aborting the TB.
199	*/
200	#define BODY_FLUSH_PENDING_WRITES() \
201	off = iemNativeRegFlushPendingWrites(pReNative, off);
202
203
204	/**
205	* Worker for the CheckIrq, CheckTimers and CheckTimersAndIrq builtins below.
206	*/
207	template<bool const a_fCheckTimers, bool const a_fCheckIrqs>
208	DECL_FORCE_INLINE(uint32_t) iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon(PIEMRECOMPILERSTATE pReNative, uint32_t off)
209	{
210	uint8_t const idxEflReg = !a_fCheckIrqs ? UINT8_MAX
211	: iemNativeRegAllocTmpForGuestEFlags(pReNative, &off, kIemNativeGstRegUse_ReadOnly,
212	RT_BIT_64(IEMLIVENESSBIT_IDX_EFL_OTHER));
213	uint8_t const idxTmpReg1 = iemNativeRegAllocTmp(pReNative, &off);
214	uint8_t const idxTmpReg2 = a_fCheckIrqs ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
215	PIEMNATIVEINSTR const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, RT_ARCH_VAL == RT_ARCH_VAL_AMD64 ? 72 : 32);
216
217	/*
218	* First we decrement the timer poll counter, if so desired.
219	*/
220	if (a_fCheckTimers)
221	{
222	# ifdef RT_ARCH_AMD64
223	/* dec [rbx + cTbsTillNextTimerPoll] */
224	pCodeBuf[off++] = 0xff;
225	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, 1, RT_UOFFSETOF(VMCPU, iem.s.cTbsTillNextTimerPoll));
226
227	/* jz ReturnBreakFF */
228	off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_e);
229
230	# elif defined(RT_ARCH_ARM64)
231	AssertCompile(RTASSERT_OFFSET_OF(VMCPU, iem.s.cTbsTillNextTimerPoll) < _4K * sizeof(uint32_t));
232	off = iemNativeEmitLoadGprFromVCpuU32Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPU, iem.s.cTbsTillNextTimerPoll));
233	pCodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxTmpReg1, idxTmpReg1, 1, false /f64Bit/);
234	off = iemNativeEmitStoreGprToVCpuU32Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPU, iem.s.cTbsTillNextTimerPoll));
235
236	/* cbz reg1, ReturnBreakFF */
237	off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg1, false /f64Bit/,
238	kIemNativeLabelType_ReturnBreakFF);
239
240	# else
241	# error "port me"
242	# endif
243	}
244
245	/*
246	* Second, check forced flags, if so desired.
247	*
248	* We OR them together to save a conditional. A trick here is that the
249	* two IRQ flags are unused in the global flags, so we can still use the
250	* resulting value to check for suppressed interrupts.
251	*/
252	if (a_fCheckIrqs)
253	{
254	/* Load VMCPU::fLocalForcedActions first and mask it. We can simplify the
255	masking by ASSUMING none of the unwanted flags are located above bit 30. */
256	uint64_t const fUnwantedCpuFFs = VMCPU_FF_PGM_SYNC_CR3
257	\| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
258	\| VMCPU_FF_TLB_FLUSH
259	\| VMCPU_FF_UNHALT;
260	AssertCompile(fUnwantedCpuFFs < RT_BIT_64(31));
261	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, idxTmpReg1, RT_UOFFSETOF(VMCPUCC, fLocalForcedActions));
262	# if defined(RT_ARCH_AMD64)
263	/* and reg1, ~fUnwantedCpuFFs */
264	pCodeBuf[off++] = idxTmpReg1 >= 8 ? X86_OP_REX_B \| X86_OP_REX_W : X86_OP_REX_W;
265	pCodeBuf[off++] = 0x81;
266	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, idxTmpReg1 & 7);
267	(uint32_t )&pCodeBuf[off] = ~(uint32_t)fUnwantedCpuFFs;
268	off += 4;
269
270	# else
271	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxTmpReg2, ~fUnwantedCpuFFs);
272	off = iemNativeEmitAndGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
273	# endif
274
275	/* OR in VM::fGlobalForcedActions. We access the member via pVCpu.
276	No need to mask anything here. Unfortunately, it's a 32-bit
277	variable, so we can't OR it directly on x86. */
278	AssertCompile(VM_FF_ALL_MASK == UINT32_MAX);
279	intptr_t const offGlobalForcedActions = (intptr_t)&pReNative->pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions
280	- (intptr_t)pReNative->pVCpu;
281	Assert((int32_t)offGlobalForcedActions == offGlobalForcedActions);
282
283	# ifdef RT_ARCH_AMD64
284	if (idxTmpReg2 >= 8)
285	pCodeBuf[off++] = X86_OP_REX_R;
286	pCodeBuf[off++] = 0x8b; /* mov */
287	off = iemNativeEmitGprByVCpuSignedDisp(pCodeBuf, off, idxTmpReg2, (int32_t)offGlobalForcedActions);
288
289	/* or reg1, reg2 */
290	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
291
292	/* jz nothing_pending */
293	uint32_t const offFixup1 = off;
294	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 64, kIemNativeInstrCond_e);
295
296	# elif defined(RT_ARCH_ARM64)
297	off = iemNativeEmitGprBySignedVCpuLdStEx(pCodeBuf, off, idxTmpReg2, (int32_t)offGlobalForcedActions,
298	kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
299	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, idxTmpReg1, idxTmpReg2);
300
301	/* cbz nothing_pending */
302	uint32_t const offFixup1 = off;
303	off = iemNativeEmitTestIfGprIsZeroOrNotZeroAndJmpToFixedEx(pCodeBuf, off, idxTmpReg1, true /f64Bit/,
304	false /fJmpIfNotZero/, off + 16);
305	# else
306	# error "port me"
307	# endif
308
309	/* More than just IRQ FFs pending? */
310	AssertCompile((VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC) == 3);
311	/* cmp reg1, 3 */
312	off = iemNativeEmitCmpGprWithImmEx(pCodeBuf, off, idxTmpReg1, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC);
313	/* ja ReturnBreakFF */
314	off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_nbe);
315
316	/*
317	* Okay, we've only got pending IRQ related FFs: Can we dispatch IRQs?
318	*
319	* ASSUME that the shadow flags are cleared when they ought to be cleared,
320	* so we can skip the RIP check.
321	*/
322	AssertCompile(CPUMCTX_INHIBIT_SHADOW < RT_BIT_32(31));
323	/* reg1 = efl & (IF \| INHIBIT_SHADOW) */
324	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxTmpReg1, idxEflReg, X86_EFL_IF \| CPUMCTX_INHIBIT_SHADOW);
325	/* reg1 ^= IF */
326	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, idxTmpReg1, X86_EFL_IF);
327
328	# ifdef RT_ARCH_AMD64
329	/* jz ReturnBreakFF */
330	off = iemNativeEmitJccTbExitEx(pReNative, pCodeBuf, off, kIemNativeLabelType_ReturnBreakFF, kIemNativeInstrCond_e);
331
332	# elif defined(RT_ARCH_ARM64)
333	/* cbz reg1, ReturnBreakFF */
334	off = iemNativeEmitTestIfGprIsZeroAndTbExitEx(pReNative, pCodeBuf, off, idxTmpReg1, false /f64Bit/,
335	kIemNativeLabelType_ReturnBreakFF);
336	# else
337	# error "port me"
338	# endif
339	/*
340	* nothing_pending:
341	*/
342	iemNativeFixupFixedJump(pReNative, offFixup1, off);
343	}
344
345	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
346
347	/*
348	* Cleanup.
349	*/
350	iemNativeRegFreeTmp(pReNative, idxTmpReg1);
351	if (a_fCheckIrqs)
352	{
353	iemNativeRegFreeTmp(pReNative, idxTmpReg2);
354	iemNativeRegFreeTmp(pReNative, idxEflReg);
355	}
356	else
357	{
358	Assert(idxTmpReg2 == UINT8_MAX);
359	Assert(idxEflReg == UINT8_MAX);
360	}
361
362	return off;
363	}
364
365
366	/**
367	* Built-in function that checks for pending interrupts that can be delivered or
368	* forced action flags.
369	*
370	* This triggers after the completion of an instruction, so EIP is already at
371	* the next instruction. If an IRQ or important FF is pending, this will return
372	* a non-zero status that stops TB execution.
373	*/
374	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckIrq)
375	{
376	BODY_FLUSH_PENDING_WRITES();
377	off = iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<false, true>(pReNative, off);
378
379	/* Note down that we've been here, so we can skip FFs + IRQ checks when
380	doing direct linking. */
381	#ifdef IEMNATIVE_WITH_LIVENESS_ANALYSIS
382	pReNative->idxLastCheckIrqCallNo = pReNative->idxCurCall;
383	RT_NOREF(pCallEntry);
384	#else
385	pReNative->idxLastCheckIrqCallNo = pCallEntry - pReNative->pTbOrg->Thrd.paCalls;
386	#endif
387
388	return off;
389	}
390
391	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckIrq)
392	{
393	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
394	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
395	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
396	RT_NOREF(pCallEntry);
397	}
398
399
400	/**
401	* Built-in function that works the cTbsTillNextTimerPoll counter on direct TB
402	* linking, like loop-jumps.
403	*/
404	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckTimers)
405	{
406	BODY_FLUSH_PENDING_WRITES();
407	RT_NOREF(pCallEntry);
408	return iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<true, false>(pReNative, off);
409	}
410
411	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckTimers)
412	{
413	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
414	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
415	RT_NOREF(pCallEntry);
416	}
417
418
419	/**
420	* Combined BltIn_CheckTimers + BltIn_CheckIrq for direct linking.
421	*/
422	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckTimersAndIrq)
423	{
424	BODY_FLUSH_PENDING_WRITES();
425	RT_NOREF(pCallEntry);
426	return iemNativeRecompFunc_BltIn_CheckTimersAndIrqsCommon<true, true>(pReNative, off);
427	}
428
429	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckTimersAndIrq)
430	{
431	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
432	IEM_LIVENESS_RAW_EFLAGS_ONE_INPUT(pOutgoing, fEflOther);
433	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
434	RT_NOREF(pCallEntry);
435	}
436
437
438	/**
439	* Built-in function checks if IEMCPU::fExec has the expected value.
440	*/
441	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckMode)
442	{
443	uint32_t const fExpectedExec = (uint32_t)pCallEntry->auParams[0];
444	uint8_t const idxTmpReg = iemNativeRegAllocTmp(pReNative, &off);
445
446	off = iemNativeEmitLoadGprFromVCpuU32(pReNative, off, idxTmpReg, RT_UOFFSETOF(VMCPUCC, iem.s.fExec));
447	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxTmpReg, IEMTB_F_KEY_MASK);
448	off = iemNativeEmitTestIfGpr32NotEqualImmAndTbExit(pReNative, off, idxTmpReg, fExpectedExec & IEMTB_F_KEY_MASK,
449	kIemNativeLabelType_ReturnBreak);
450	iemNativeRegFreeTmp(pReNative, idxTmpReg);
451
452	/* Maintain the recompiler fExec state. */
453	pReNative->fExec = fExpectedExec & IEMTB_F_IEM_F_MASK;
454	return off;
455	}
456
457	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckMode)
458	{
459	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
460	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
461	RT_NOREF(pCallEntry);
462	}
463
464
465	/**
466	* Sets idxTbCurInstr in preparation of raising an exception or aborting the TB.
467	*/
468	/** @todo Optimize this, so we don't set the same value more than once. Just
469	* needs some tracking. */
470	#ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
471	# define BODY_SET_CUR_INSTR() \
472	off = iemNativeEmitStoreImmToVCpuU8(pReNative, off, pCallEntry->idxInstr, RT_UOFFSETOF(VMCPUCC, iem.s.idxTbCurInstr))
473	#else
474	# define BODY_SET_CUR_INSTR() ((void)0)
475	#endif
476
477
478	/**
479	* Macro that emits the 16/32-bit CS.LIM check.
480	*/
481	#define BODY_CHECK_CS_LIM(a_cbInstr) \
482	off = iemNativeEmitBltInCheckCsLim(pReNative, off, (a_cbInstr))
483
484	#define LIVENESS_CHECK_CS_LIM(a_pOutgoing) \
485	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS)
486
487	DECL_FORCE_INLINE(uint32_t)
488	iemNativeEmitBltInCheckCsLim(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cbInstr)
489	{
490	Assert(cbInstr > 0);
491	Assert(cbInstr < 16);
492	#ifdef VBOX_STRICT
493	off = iemNativeEmitMarker(pReNative, off, 0x80000001);
494	#endif
495
496	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
497	Assert(pReNative->Core.offPc == 0);
498	#endif
499
500	/*
501	* We need CS.LIM and RIP here. When cbInstr is larger than 1, we also need
502	* a temporary register for calculating the last address of the instruction.
503	*
504	* The calculation and comparisons are 32-bit. We ASSUME that the incoming
505	* RIP isn't totally invalid, i.e. that any jump/call/ret/iret instruction
506	* that last updated EIP here checked it already, and that we're therefore
507	* safe in the 32-bit wrap-around scenario to only check that the last byte
508	* is within CS.LIM. In the case of instruction-by-instruction advancing
509	* up to a EIP wrap-around, we know that CS.LIM is 4G-1 because the limit
510	* must be using 4KB granularity and the previous instruction was fine.
511	*/
512	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
513	kIemNativeGstRegUse_ReadOnly);
514	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
515	kIemNativeGstRegUse_ReadOnly);
516	#ifdef RT_ARCH_AMD64
517	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
518	#elif defined(RT_ARCH_ARM64)
519	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
520	#else
521	# error "Port me"
522	#endif
523
524	if (cbInstr != 1)
525	{
526	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
527
528	/*
529	* 1. idxRegTmp = idxRegPc + cbInstr;
530	* 2. if idxRegTmp > idxRegCsLim then raise #GP(0).
531	*/
532	#ifdef RT_ARCH_AMD64
533	/* 1. lea tmp32, [Pc + cbInstr - 1] */
534	if (idxRegTmp >= 8 \|\| idxRegPc >= 8)
535	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegPc < 8 ? 0 : X86_OP_REX_B);
536	pbCodeBuf[off++] = 0x8d;
537	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, idxRegTmp & 7, idxRegPc & 7);
538	if ((idxRegPc & 7) == X86_GREG_xSP)
539	pbCodeBuf[off++] = X86_SIB_MAKE(idxRegPc & 7, 4 /no index/, 0);
540	pbCodeBuf[off++] = cbInstr - 1;
541
542	/* 2. cmp tmp32(r), CsLim(r/m). */
543	if (idxRegTmp >= 8 \|\| idxRegCsLim >= 8)
544	pbCodeBuf[off++] = (idxRegTmp < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
545	pbCodeBuf[off++] = 0x3b;
546	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegTmp & 7, idxRegCsLim & 7);
547
548	#elif defined(RT_ARCH_ARM64)
549	/* 1. add tmp32, Pc, #cbInstr-1 */
550	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /fSub/, idxRegTmp, idxRegPc, cbInstr - 1, false /f64Bit/);
551	/* 2. cmp tmp32, CsLim */
552	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegTmp, idxRegCsLim,
553	false /f64Bit/, true /fSetFlags/);
554
555	#endif
556	iemNativeRegFreeTmp(pReNative, idxRegTmp);
557	}
558	else
559	{
560	/*
561	* Here we can skip step 1 and compare PC and CS.LIM directly.
562	*/
563	#ifdef RT_ARCH_AMD64
564	/* 2. cmp eip(r), CsLim(r/m). */
565	if (idxRegPc >= 8 \|\| idxRegCsLim >= 8)
566	pbCodeBuf[off++] = (idxRegPc < 8 ? 0 : X86_OP_REX_R) \| (idxRegCsLim < 8 ? 0 : X86_OP_REX_B);
567	pbCodeBuf[off++] = 0x3b;
568	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, idxRegPc & 7, idxRegCsLim & 7);
569
570	#elif defined(RT_ARCH_ARM64)
571	/* 2. cmp Pc, CsLim */
572	pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /fSub/, ARMV8_A64_REG_XZR, idxRegPc, idxRegCsLim,
573	false /f64Bit/, true /fSetFlags/);
574
575	#endif
576	}
577
578	/* 3. Jump if greater. */
579	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_RaiseGp0);
580
581	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
582	iemNativeRegFreeTmp(pReNative, idxRegPc);
583	return off;
584	}
585
586
587	/**
588	* Macro that considers whether we need CS.LIM checking after a branch or
589	* crossing over to a new page.
590	*/
591	#define BODY_CONSIDER_CS_LIM_CHECKING(a_pTb, a_cbInstr) \
592	RT_NOREF(a_cbInstr); \
593	off = iemNativeEmitBltInConsiderLimChecking(pReNative, off)
594
595	#define LIVENESS_CONSIDER_CS_LIM_CHECKING(a_pOutgoing) \
596	IEM_LIVENESS_RAW_SEG_LIMIT_INPUT(a_pOutgoing, X86_SREG_CS); \
597	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS)
598
599	DECL_FORCE_INLINE(uint32_t)
600	iemNativeEmitBltInConsiderLimChecking(PIEMRECOMPILERSTATE pReNative, uint32_t off)
601	{
602	#ifdef VBOX_STRICT
603	off = iemNativeEmitMarker(pReNative, off, 0x80000002);
604	#endif
605
606	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
607	Assert(pReNative->Core.offPc == 0);
608	#endif
609
610	/*
611	* This check must match the ones in the iem in iemGetTbFlagsForCurrentPc
612	* exactly:
613	*
614	* int64_t const offFromLim = (int64_t)pVCpu->cpum.GstCtx.cs.u32Limit - (int64_t)pVCpu->cpum.GstCtx.eip;
615	* if (offFromLim >= X86_PAGE_SIZE + 16 - (int32_t)(pVCpu->cpum.GstCtx.cs.u64Base & GUEST_PAGE_OFFSET_MASK))
616	* return fRet;
617	* return fRet \| IEMTB_F_CS_LIM_CHECKS;
618	*
619	*
620	* We need EIP, CS.LIM and CS.BASE here.
621	*/
622
623	/* Calculate the offFromLim first: */
624	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
625	kIemNativeGstRegUse_ReadOnly);
626	uint8_t const idxRegCsLim = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS),
627	kIemNativeGstRegUse_ReadOnly);
628	uint8_t const idxRegLeft = iemNativeRegAllocTmp(pReNative, &off);
629
630	#ifdef RT_ARCH_ARM64
631	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
632	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegLeft, idxRegCsLim, idxRegPc);
633	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
634	#else
635	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegLeft, idxRegCsLim);
636	off = iemNativeEmitSubTwoGprs(pReNative, off, idxRegLeft, idxRegPc);
637	#endif
638
639	iemNativeRegFreeTmp(pReNative, idxRegCsLim);
640	iemNativeRegFreeTmp(pReNative, idxRegPc);
641
642	/* Calculate the threshold level (right side). */
643	uint8_t const idxRegCsBase = iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
644	kIemNativeGstRegUse_ReadOnly);
645	uint8_t const idxRegRight = iemNativeRegAllocTmp(pReNative, &off);
646
647	#ifdef RT_ARCH_ARM64
648	pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
649	Assert(Armv8A64ConvertImmRImmS2Mask32(11, 0) == GUEST_PAGE_OFFSET_MASK);
650	pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(idxRegRight, idxRegCsBase, 11, 0, false /f64Bit/);
651	pu32CodeBuf[off++] = Armv8A64MkInstrNeg(idxRegRight);
652	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
653	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegRight, idxRegRight, (X86_PAGE_SIZE + 16) / 2);
654	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
655
656	#else
657	off = iemNativeEmitLoadGprImm32(pReNative, off, idxRegRight, GUEST_PAGE_OFFSET_MASK);
658	off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, idxRegRight, idxRegCsBase);
659	off = iemNativeEmitNegGpr(pReNative, off, idxRegRight);
660	off = iemNativeEmitAddGprImm(pReNative, off, idxRegRight, X86_PAGE_SIZE + 16);
661	#endif
662
663	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
664
665	/* Compare the two and jump out if we're too close to the limit. */
666	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegLeft, idxRegRight);
667	off = iemNativeEmitJlTbExit(pReNative, off, kIemNativeLabelType_NeedCsLimChecking);
668
669	iemNativeRegFreeTmp(pReNative, idxRegRight);
670	iemNativeRegFreeTmp(pReNative, idxRegLeft);
671	return off;
672	}
673
674
675
676	/**
677	* Macro that implements opcode (re-)checking.
678	*/
679	#define BODY_CHECK_OPCODES(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
680	RT_NOREF(a_cbInstr); \
681	off = iemNativeEmitBltInCheckOpcodes(pReNative, off, (a_pTb), (a_idxRange), (a_offRange))
682
683	#define LIVENESS_CHECK_OPCODES(a_pOutgoing) ((void)0)
684
685	#if 0 /* debugging aid */
686	bool g_fBpOnObsoletion = false;
687	# define BP_ON_OBSOLETION g_fBpOnObsoletion
688	#else
689	# define BP_ON_OBSOLETION 0
690	#endif
691
692	DECL_FORCE_INLINE(uint32_t)
693	iemNativeEmitBltInCheckOpcodes(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint16_t offRange)
694	{
695	Assert(idxRange < pTb->cRanges && pTb->cRanges <= RT_ELEMENTS(pTb->aRanges));
696	Assert(offRange < pTb->aRanges[idxRange].cbOpcodes);
697	#ifdef VBOX_STRICT
698	off = iemNativeEmitMarker(pReNative, off, 0x80000003);
699	#endif
700
701	/*
702	* Where to start and how much to compare.
703	*
704	* Looking at the ranges produced when r160746 was running a DOS VM with TB
705	* logging, the ranges can be anything from 1 byte to at least 0x197 bytes,
706	* with the 6, 5, 4, 7, 8, 40, 3, 2, 9 and 10 being the top 10 in the sample.
707	*
708	* The top 10 for the early boot phase of a 64-bit debian 9.4 VM: 5, 9, 8,
709	* 12, 10, 11, 6, 13, 15 and 16. Max 0x359 bytes. Same revision as above.
710	*/
711	uint16_t offPage = pTb->aRanges[idxRange].offPhysPage + offRange;
712	uint16_t cbLeft = pTb->aRanges[idxRange].cbOpcodes - offRange;
713	Assert(cbLeft > 0);
714	uint8_t const *pbOpcodes = &pTb->pabOpcodes[pTb->aRanges[idxRange].offOpcodes + offRange];
715	uint32_t offConsolidatedJump = UINT32_MAX;
716
717	#ifdef RT_ARCH_AMD64
718	/* AMD64/x86 offers a bunch of options. Smaller stuff will can be
719	completely inlined, for larger we use REPE CMPS. */
720	# define CHECK_OPCODES_CMP_IMMXX(a_idxReg, a_bOpcode) /* cost: 3 bytes */ do { \
721	pbCodeBuf[off++] = a_bOpcode; \
722	Assert(offPage < 127); \
723	pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 7, a_idxReg); \
724	pbCodeBuf[off++] = RT_BYTE1(offPage); \
725	} while (0)
726
727	# define NEAR_JMP_SIZE 5
728
729	# define CHECK_OPCODES_CMP_JMP() /* cost: 7 bytes first time, then 2 bytes */ do { \
730	if (offConsolidatedJump != UINT32_MAX) \
731	{ \
732	int32_t const offDisp = (int32_t)offConsolidatedJump - (int32_t)(off + 2); \
733	Assert(offDisp >= -128); \
734	pbCodeBuf[off++] = 0x75; /* jnz near */ \
735	pbCodeBuf[off++] = (uint8_t)offDisp; \
736	} \
737	else \
738	{ \
739	pbCodeBuf[off++] = 0x74; /* jz near +NEAR_JMP_SIZE */ \
740	pbCodeBuf[off++] = NEAR_JMP_SIZE + BP_ON_OBSOLETION; \
741	offConsolidatedJump = off; \
742	if (BP_ON_OBSOLETION) pbCodeBuf[off++] = 0xcc; \
743	off = iemNativeEmitTbExitEx(pReNative, pbCodeBuf, off, kIemNativeLabelType_ObsoleteTb); \
744	} \
745	} while (0)
746
747	# define CHECK_OPCODES_CMP_IMM32(a_idxReg) /* cost: 3+4+2 = 9 */ do { \
748	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
749	pbCodeBuf[off++] = *pbOpcodes++; \
750	pbCodeBuf[off++] = *pbOpcodes++; \
751	pbCodeBuf[off++] = *pbOpcodes++; \
752	pbCodeBuf[off++] = *pbOpcodes++; \
753	cbLeft -= 4; \
754	offPage += 4; \
755	CHECK_OPCODES_CMP_JMP(); \
756	} while (0)
757
758	# define CHECK_OPCODES_CMP_IMM16(a_idxReg) /* cost: 1+3+2+2 = 8 */ do { \
759	pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP; \
760	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x81); \
761	pbCodeBuf[off++] = *pbOpcodes++; \
762	pbCodeBuf[off++] = *pbOpcodes++; \
763	cbLeft -= 2; \
764	offPage += 2; \
765	CHECK_OPCODES_CMP_JMP(); \
766	} while (0)
767
768	# define CHECK_OPCODES_CMP_IMM8(a_idxReg) /* cost: 3+1+2 = 6 */ do { \
769	CHECK_OPCODES_CMP_IMMXX(a_idxReg, 0x80); \
770	pbCodeBuf[off++] = *pbOpcodes++; \
771	cbLeft -= 1; \
772	offPage += 1; \
773	CHECK_OPCODES_CMP_JMP(); \
774	} while (0)
775
776	# define CHECK_OPCODES_CMPSX(a_bOpcode, a_cbToSubtract, a_bPrefix) /* cost: 2+2 = 4 */ do { \
777	if (a_bPrefix) \
778	pbCodeBuf[off++] = (a_bPrefix); \
779	pbCodeBuf[off++] = (a_bOpcode); \
780	CHECK_OPCODES_CMP_JMP(); \
781	cbLeft -= (a_cbToSubtract); \
782	} while (0)
783
784	# define CHECK_OPCODES_ECX_IMM(a_uValue) /* cost: 5 */ do { \
785	pbCodeBuf[off++] = 0xb8 + X86_GREG_xCX; \
786	pbCodeBuf[off++] = RT_BYTE1(a_uValue); \
787	pbCodeBuf[off++] = RT_BYTE2(a_uValue); \
788	pbCodeBuf[off++] = RT_BYTE3(a_uValue); \
789	pbCodeBuf[off++] = RT_BYTE4(a_uValue); \
790	} while (0)
791
792	if (cbLeft <= 24)
793	{
794	uint8_t const idxRegTmp = iemNativeRegAllocTmpEx(pReNative, &off,
795	( RT_BIT_32(X86_GREG_xAX)
796	\| RT_BIT_32(X86_GREG_xCX)
797	\| RT_BIT_32(X86_GREG_xDX)
798	\| RT_BIT_32(X86_GREG_xBX)
799	\| RT_BIT_32(X86_GREG_xSI)
800	\| RT_BIT_32(X86_GREG_xDI))
801	& ~IEMNATIVE_REG_FIXED_MASK); /* pick reg not requiring rex prefix */
802	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf));
803	if (offPage >= 128 - cbLeft)
804	{
805	off = iemNativeEmitAddGprImm(pReNative, off, idxRegTmp, offPage & ~(uint16_t)3);
806	offPage &= 3;
807	}
808
809	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 14 + 54 + 8 + 6 + BP_ON_OBSOLETION /* = 88 */);
810
811	if (cbLeft > 8)
812	switch (offPage & 3)
813	{
814	case 0:
815	break;
816	case 1: /* cost: 6 + 8 = 14 */
817	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
818	RT_FALL_THRU();
819	case 2: /* cost: 8 */
820	CHECK_OPCODES_CMP_IMM16(idxRegTmp);
821	break;
822	case 3: /* cost: 6 */
823	CHECK_OPCODES_CMP_IMM8(idxRegTmp);
824	break;
825	}
826
827	while (cbLeft >= 4)
828	CHECK_OPCODES_CMP_IMM32(idxRegTmp); /* max iteration: 24/4 = 6; --> cost: 6 * 9 = 54 */
829
830	if (cbLeft >= 2)
831	CHECK_OPCODES_CMP_IMM16(idxRegTmp); /* cost: 8 */
832	if (cbLeft)
833	CHECK_OPCODES_CMP_IMM8(idxRegTmp); /* cost: 6 */
834
835	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
836	iemNativeRegFreeTmp(pReNative, idxRegTmp);
837	}
838	else
839	{
840	/* RDI = &pbInstrBuf[offPage] */
841	uint8_t const idxRegDi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xDI));
842	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegDi, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
843	if (offPage != 0)
844	off = iemNativeEmitAddGprImm(pReNative, off, idxRegDi, offPage);
845
846	/* RSI = pbOpcodes */
847	uint8_t const idxRegSi = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xSI));
848	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegSi, (uintptr_t)pbOpcodes);
849
850	/* RCX = counts. */
851	uint8_t const idxRegCx = iemNativeRegAllocTmpEx(pReNative, &off, RT_BIT_32(X86_GREG_xCX));
852
853	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6 + 10 + 5 + 5 + 3 + 4 + 3 + BP_ON_OBSOLETION /= 36/);
854
855	/** @todo profile and optimize this further. Maybe an idea to align by
856	* offPage if the two cannot be reconsidled. */
857	/* Align by the page offset, so that at least one of the accesses are naturally aligned. */
858	switch (offPage & 7) /* max cost: 10 */
859	{
860	case 0:
861	break;
862	case 1: /* cost: 3+4+3 = 10 */
863	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
864	RT_FALL_THRU();
865	case 2: /* cost: 4+3 = 7 */
866	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
867	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
868	break;
869	case 3: /* cost: 3+3 = 6 */
870	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
871	RT_FALL_THRU();
872	case 4: /* cost: 3 */
873	CHECK_OPCODES_CMPSX(0xa7, 4, 0);
874	break;
875	case 5: /* cost: 3+4 = 7 */
876	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
877	RT_FALL_THRU();
878	case 6: /* cost: 4 */
879	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP);
880	break;
881	case 7: /* cost: 3 */
882	CHECK_OPCODES_CMPSX(0xa6, 1, 0);
883	break;
884	}
885
886	/* Compare qwords: */
887	uint32_t const cQWords = cbLeft >> 3;
888	CHECK_OPCODES_ECX_IMM(cQWords); /* cost: 5 */
889
890	pbCodeBuf[off++] = X86_OP_PRF_REPZ; /* cost: 5 */
891	CHECK_OPCODES_CMPSX(0xa7, 0, X86_OP_REX_W);
892	cbLeft &= 7;
893
894	if (cbLeft & 4)
895	CHECK_OPCODES_CMPSX(0xa7, 4, 0); /* cost: 3 */
896	if (cbLeft & 2)
897	CHECK_OPCODES_CMPSX(0xa7, 2, X86_OP_PRF_SIZE_OP); /* cost: 4 */
898	if (cbLeft & 1)
899	CHECK_OPCODES_CMPSX(0xa6, 1, 0); /* cost: 3 */
900
901	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
902	iemNativeRegFreeTmp(pReNative, idxRegCx);
903	iemNativeRegFreeTmp(pReNative, idxRegSi);
904	iemNativeRegFreeTmp(pReNative, idxRegDi);
905	}
906
907	#elif defined(RT_ARCH_ARM64)
908	/* We need pbInstrBuf in a register, whatever we do. */
909	uint8_t const idxRegSrc1Ptr = iemNativeRegAllocTmp(pReNative, &off);
910	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegSrc1Ptr, RT_UOFFSETOF(VMCPU, iem.s.pbInstrBuf));
911
912	/* We also need at least one more register for holding bytes & words we
913	load via pbInstrBuf. */
914	uint8_t const idxRegSrc1Val = iemNativeRegAllocTmp(pReNative, &off);
915
916	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
917
918	/* One byte compare can be done with the opcode byte as an immediate. We'll
919	do this to uint16_t align src1. */
920	bool fPendingJmp = RT_BOOL(offPage & 1);
921	if (fPendingJmp)
922	{
923	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
924	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes++, false /f64Bit*/);
925	offPage += 1;
926	cbLeft -= 1;
927	}
928
929	if (cbLeft > 0)
930	{
931	/* We need a register for holding the opcode bytes we're comparing with,
932	as CCMP only has a 5-bit immediate form and thus cannot hold bytes. */
933	uint8_t const idxRegSrc2Val = iemNativeRegAllocTmp(pReNative, &off);
934
935	/* Word (uint32_t) aligning the src1 pointer is best done using a 16-bit constant load. */
936	if ((offPage & 3) && cbLeft >= 2)
937	{
938	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 2);
939	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
940	if (fPendingJmp)
941	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
942	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
943	else
944	{
945	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
946	fPendingJmp = true;
947	}
948	pbOpcodes += 2;
949	offPage += 2;
950	cbLeft -= 2;
951	}
952
953	/* DWord (uint64_t) aligning the src2 pointer. We use a 32-bit constant here for simplicitly. */
954	if ((offPage & 7) && cbLeft >= 4)
955	{
956	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr, offPage / 4);
957	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
958	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
959	if (fPendingJmp)
960	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
961	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
962	else
963	{
964	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
965	fPendingJmp = true;
966	}
967	pbOpcodes += 4;
968	offPage += 4;
969	cbLeft -= 4;
970	}
971
972	/*
973	* If we've got 16 bytes or more left, switch to memcmp-style.
974	*/
975	if (cbLeft >= 16)
976	{
977	/* We need a pointer to the copy of the original opcode bytes. */
978	uint8_t const idxRegSrc2Ptr = iemNativeRegAllocTmp(pReNative, &off);
979	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Ptr, (uintptr_t)pbOpcodes);
980
981	/* If there are more than 32 bytes to compare we create a loop, for
982	which we'll need a loop register. */
983	if (cbLeft >= 64)
984	{
985	if (fPendingJmp)
986	{
987	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
988	kArmv8InstrCond_Ne);
989	fPendingJmp = false;
990	}
991
992	uint8_t const idxRegLoop = iemNativeRegAllocTmp(pReNative, &off);
993	uint16_t const cLoops = cbLeft / 32;
994	cbLeft = cbLeft % 32;
995	pbOpcodes += cLoops * 32;
996	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegLoop, cLoops);
997
998	if (offPage != 0) /** @todo optimize out this instruction. */
999	{
1000	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, offPage);
1001	offPage = 0;
1002	}
1003
1004	uint32_t const offLoopStart = off;
1005	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 0);
1006	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 0);
1007	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
1008
1009	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 1);
1010	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 1);
1011	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1012	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1013
1014	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 2);
1015	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 2);
1016	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1017	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1018
1019	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr, 3);
1020	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val, idxRegSrc2Ptr, 3);
1021	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1022	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1023
1024	off = iemNativeEmitJccTbExitEx(pReNative, pu32CodeBuf, off, kIemNativeLabelType_ObsoleteTb,
1025	kArmv8InstrCond_Ne);
1026
1027	/* Advance and loop. */
1028	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc1Ptr, idxRegSrc1Ptr, 0x20);
1029	pu32CodeBuf[off++] = Armv8A64MkInstrAddUImm12(idxRegSrc2Ptr, idxRegSrc2Ptr, 0x20);
1030	pu32CodeBuf[off++] = Armv8A64MkInstrSubUImm12(idxRegLoop, idxRegLoop, 1, false /f64Bit/, true /fSetFlags/);
1031	pu32CodeBuf[off] = Armv8A64MkInstrBCond(kArmv8InstrCond_Ne, (int32_t)offLoopStart - (int32_t)off);
1032	off++;
1033
1034	iemNativeRegFreeTmp(pReNative, idxRegLoop);
1035	}
1036
1037	/* Deal with any remaining dwords (uint64_t). There can be up to
1038	three if we looped and four if we didn't. */
1039	uint32_t offSrc2 = 0;
1040	while (cbLeft >= 8)
1041	{
1042	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val,
1043	idxRegSrc1Ptr, offPage / 8);
1044	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc2Val,
1045	idxRegSrc2Ptr, offSrc2 / 8);
1046	if (fPendingJmp)
1047	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1048	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq);
1049	else
1050	{
1051	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val);
1052	fPendingJmp = true;
1053	}
1054	pbOpcodes += 8;
1055	offPage += 8;
1056	offSrc2 += 8;
1057	cbLeft -= 8;
1058	}
1059
1060	iemNativeRegFreeTmp(pReNative, idxRegSrc2Ptr);
1061	/* max cost thus far: memcmp-loop=43 vs memcmp-no-loop=30 */
1062	}
1063	/*
1064	* Otherwise, we compare with constants and merge with the general mop-up.
1065	*/
1066	else
1067	{
1068	while (cbLeft >= 8)
1069	{
1070	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Dword, idxRegSrc1Val, idxRegSrc1Ptr,
1071	offPage / 8);
1072	off = iemNativeEmitLoadGprImmEx(pu32CodeBuf, off, idxRegSrc2Val,
1073	RT_MAKE_U64_FROM_MSB_U8(pbOpcodes[7], pbOpcodes[6], pbOpcodes[5], pbOpcodes[4],
1074	pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
1075	if (fPendingJmp)
1076	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1077	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, true /f64Bit/);
1078	else
1079	{
1080	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, true /f64Bit/);
1081	fPendingJmp = true;
1082	}
1083	pbOpcodes += 8;
1084	offPage += 8;
1085	cbLeft -= 8;
1086	}
1087	/* max cost thus far: 21 */
1088	}
1089
1090	/* Deal with any remaining bytes (7 or less). */
1091	Assert(cbLeft < 8);
1092	if (cbLeft >= 4)
1093	{
1094	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Word, idxRegSrc1Val, idxRegSrc1Ptr,
1095	offPage / 4);
1096	off = iemNativeEmitLoadGpr32ImmEx(pu32CodeBuf, off, idxRegSrc2Val,
1097	RT_MAKE_U32_FROM_MSB_U8(pbOpcodes[3], pbOpcodes[2], pbOpcodes[1], pbOpcodes[0]));
1098	if (fPendingJmp)
1099	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1100	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1101	else
1102	{
1103	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
1104	fPendingJmp = true;
1105	}
1106	pbOpcodes += 4;
1107	offPage += 4;
1108	cbLeft -= 4;
1109
1110	}
1111
1112	if (cbLeft >= 2)
1113	{
1114	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Half, idxRegSrc1Val, idxRegSrc1Ptr,
1115	offPage / 2);
1116	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, RT_MAKE_U16(pbOpcodes[0], pbOpcodes[1]));
1117	if (fPendingJmp)
1118	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1119	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1120	else
1121	{
1122	pu32CodeBuf[off++] = Armv8A64MkInstrCmpReg(idxRegSrc1Val, idxRegSrc2Val, false /f64Bit/);
1123	fPendingJmp = true;
1124	}
1125	pbOpcodes += 2;
1126	offPage += 2;
1127	cbLeft -= 2;
1128	}
1129
1130	if (cbLeft > 0)
1131	{
1132	Assert(cbLeft == 1);
1133	pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_Ld_Byte, idxRegSrc1Val, idxRegSrc1Ptr, offPage);
1134	if (fPendingJmp)
1135	{
1136	pu32CodeBuf[off++] = Armv8A64MkInstrMovZ(idxRegSrc2Val, pbOpcodes[0]);
1137	pu32CodeBuf[off++] = Armv8A64MkInstrCCmpReg(idxRegSrc1Val, idxRegSrc2Val,
1138	ARMA64_NZCV_F_N0_Z0_C0_V0, kArmv8InstrCond_Eq, false /f64Bit/);
1139	}
1140	else
1141	{
1142	pu32CodeBuf[off++] = Armv8A64MkInstrCmpUImm12(idxRegSrc1Val, pbOpcodes[0], false /f64Bit/);
1143	fPendingJmp = true;
1144	}
1145	pbOpcodes += 1;
1146	offPage += 1;
1147	cbLeft -= 1;
1148	}
1149
1150	iemNativeRegFreeTmp(pReNative, idxRegSrc2Val);
1151	}
1152	Assert(cbLeft == 0);
1153
1154	/*
1155	* Finally, the branch on difference.
1156	*/
1157	if (fPendingJmp)
1158	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_ObsoleteTb);
1159
1160	RT_NOREF(pu32CodeBuf, cbLeft, offPage, pbOpcodes, offConsolidatedJump);
1161
1162	/* max costs: memcmp-loop=54; memcmp-no-loop=41; only-src1-ptr=32 */
1163	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1164	iemNativeRegFreeTmp(pReNative, idxRegSrc1Val);
1165	iemNativeRegFreeTmp(pReNative, idxRegSrc1Ptr);
1166
1167	#else
1168	# error "Port me"
1169	#endif
1170	return off;
1171	}
1172
1173
1174
1175	/**
1176	* Macro that implements PC check after a conditional branch.
1177	*/
1178	#define BODY_CHECK_PC_AFTER_BRANCH(a_pTb, a_idxRange, a_offRange, a_cbInstr) \
1179	RT_NOREF(a_cbInstr); \
1180	off = iemNativeEmitBltInCheckPcAfterBranch(pReNative, off, a_pTb, a_idxRange, a_offRange)
1181
1182	#define LIVENESS_CHECK_PC_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1183	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1184	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1185	else do { } while (0)
1186
1187	DECL_FORCE_INLINE(uint32_t)
1188	iemNativeEmitBltInCheckPcAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb,
1189	uint8_t idxRange, uint16_t offRange)
1190	{
1191	#ifdef VBOX_STRICT
1192	off = iemNativeEmitMarker(pReNative, off, 0x80000004);
1193	#endif
1194
1195	#ifdef IEMNATIVE_WITH_DELAYED_PC_UPDATING
1196	Assert(pReNative->Core.offPc == 0);
1197	#endif
1198
1199	/*
1200	* The GCPhysRangePageWithOffset value in the threaded function is a fixed
1201	* constant for us here.
1202	*
1203	* We can pretend that iem.s.cbInstrBufTotal is X86_PAGE_SIZE here, because
1204	* it serves no purpose as a CS.LIM, if that's needed we've just performed
1205	* it, and as long as we don't implement code TLB reload code here there is
1206	* no point in checking that the TLB data we're using is still valid.
1207	*
1208	* What we to do is.
1209	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1210	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1211	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1212	* we're in the wrong spot and need to find a new TB.
1213	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1214	* GCPhysRangePageWithOffset constant mentioned above.
1215	*
1216	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1217	* in 64-bit code or flat 32-bit.
1218	*/
1219
1220	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1221	the temp register, so we don't accidentally clobber something we'll be
1222	needing again immediately. This is why we get idxRegCsBase here. */
1223	uint8_t const idxRegPc = iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1224	kIemNativeGstRegUse_ReadOnly);
1225	uint8_t const idxRegCsBase = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX
1226	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1227	kIemNativeGstRegUse_ReadOnly);
1228
1229	uint8_t const idxRegTmp = iemNativeRegAllocTmp(pReNative, &off);
1230
1231	#ifdef VBOX_STRICT
1232	/* Do assertions before idxRegTmp contains anything. */
1233	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1234	# ifdef RT_ARCH_AMD64
1235	{
1236	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1237	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1238	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1239	{
1240	/* cmp r/m64, imm8 */
1241	pbCodeBuf[off++] = X86_OP_REX_W;
1242	pbCodeBuf[off++] = 0x83;
1243	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1244	pbCodeBuf[off++] = 0;
1245	/* je rel8 */
1246	pbCodeBuf[off++] = 0x74;
1247	pbCodeBuf[off++] = 1;
1248	/* int3 */
1249	pbCodeBuf[off++] = 0xcc;
1250
1251	}
1252
1253	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1254	/* test r/m64, imm32 */
1255	pbCodeBuf[off++] = X86_OP_REX_W;
1256	pbCodeBuf[off++] = 0xf7;
1257	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1258	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1259	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1260	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1261	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1262	/* jz rel8 */
1263	pbCodeBuf[off++] = 0x74;
1264	pbCodeBuf[off++] = 1;
1265	/* int3 */
1266	pbCodeBuf[off++] = 0xcc;
1267	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1268	}
1269	# else
1270
1271	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1272	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1273	{
1274	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1275	# ifdef RT_ARCH_ARM64
1276	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1277	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1278	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2004);
1279	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1280	# else
1281	# error "Port me!"
1282	# endif
1283	}
1284	# endif
1285
1286	#endif /* VBOX_STRICT */
1287
1288	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1289	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1290	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1291	{
1292	#ifdef RT_ARCH_ARM64
1293	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1294	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1295	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1296	#else
1297	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1298	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1299	#endif
1300	}
1301	else
1302	{
1303	#ifdef RT_ARCH_ARM64
1304	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1305	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1306	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1307	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1308	#else
1309	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1310	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1311	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1312	#endif
1313	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1314	}
1315	iemNativeRegFreeTmp(pReNative, idxRegPc);
1316
1317	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal. */
1318	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1319	off = iemNativeEmitJaTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1320
1321	/* 4. Add iem.s.GCPhysInstrBuf and compare with GCPhysRangePageWithOffset. */
1322	#ifdef RT_ARCH_AMD64
1323	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1324	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1325	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1326	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1327	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1328
1329	#elif defined(RT_ARCH_ARM64)
1330	uint8_t const idxRegTmp2 = iemNativeRegAllocTmp(pReNative, &off);
1331
1332	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1333	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1334	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1335	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1336
1337	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1338	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1339	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1340	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1341	# endif
1342	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1343	#else
1344	# error "Port me"
1345	#endif
1346
1347	RTGCPHYS const GCPhysRangePageWithOffset = ( iemTbGetRangePhysPageAddr(pTb, idxRange)
1348	\| pTb->aRanges[idxRange].offPhysPage)
1349	+ offRange;
1350	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegTmp, GCPhysRangePageWithOffset,
1351	kIemNativeLabelType_CheckBranchMiss);
1352
1353	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1354	return off;
1355	}
1356
1357
1358	/**
1359	* Macro that implements TLB loading and updating pbInstrBuf updating for an
1360	* instruction crossing into a new page.
1361	*
1362	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1363	*/
1364	#define BODY_LOAD_TLB_FOR_NEW_PAGE(a_pTb, a_offInstr, a_idxRange, a_cbInstr) \
1365	RT_NOREF(a_cbInstr); \
1366	off = iemNativeEmitBltLoadTlbForNewPage(pReNative, off, pTb, a_idxRange, a_offInstr)
1367
1368	#define LIVENESS_LOAD_TLB_FOR_NEW_PAGE(a_pOutgoing, a_pCallEntry) \
1369	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1370	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1371	else do { } while (0)
1372
1373	DECL_FORCE_INLINE(uint32_t)
1374	iemNativeEmitBltLoadTlbForNewPage(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange, uint8_t offInstr)
1375	{
1376	#ifdef VBOX_STRICT
1377	off = iemNativeEmitMarker(pReNative, off, 0x80000005);
1378	#endif
1379
1380	/*
1381	* Define labels and allocate the register for holding the GCPhys of the new page.
1382	*/
1383	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1384	uint32_t const idxRegGCPhys = iemNativeRegAllocTmp(pReNative, &off);
1385	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, IEM_F_MODE_X86_IS_FLAT(pReNative->fExec), &off);
1386	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1387	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1388	: UINT32_MAX;
1389
1390	//off = iemNativeEmitBrk(pReNative, off, 0x1111);
1391
1392	/*
1393	* Jump to the TLB lookup code.
1394	*/
1395	if (!TlbState.fSkip)
1396	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1397
1398	/*
1399	* TlbMiss:
1400	*
1401	* Call iemNativeHlpMemCodeNewPageTlbMissWithOff to do the work.
1402	*/
1403	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1404
1405	/* Save variables in volatile registers. */
1406	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegGCPhys);
1407	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1408
1409	/* IEMNATIVE_CALL_ARG1_GREG = offInstr */
1410	off = iemNativeEmitLoadGpr8Imm(pReNative, off, IEMNATIVE_CALL_ARG1_GREG, offInstr);
1411
1412	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1413	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1414
1415	/* Done setting up parameters, make the call. */
1416	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMissWithOff);
1417
1418	/* Move the result to the right register. */
1419	if (idxRegGCPhys != IEMNATIVE_CALL_RET_GREG)
1420	off = iemNativeEmitLoadGprFromGpr(pReNative, off, idxRegGCPhys, IEMNATIVE_CALL_RET_GREG);
1421
1422	/* Restore variables and guest shadow registers to volatile registers. */
1423	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1424	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off, TlbState.getActiveRegsWithShadows(true /fCode/));
1425
1426	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1427	if (!TlbState.fSkip)
1428	{
1429	/* end of TlbMiss - Jump to the done label. */
1430	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1431	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1432
1433	/*
1434	* TlbLookup:
1435	*/
1436	off = iemNativeEmitTlbLookup<false>(pReNative, off, &TlbState,
1437	IEM_F_MODE_X86_IS_FLAT(pReNative->fExec) ? UINT8_MAX : X86_SREG_CS,
1438	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1439	idxLabelTlbLookup, idxLabelTlbMiss, idxRegGCPhys, offInstr);
1440
1441	# ifdef IEM_WITH_TLB_STATISTICS
1442	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1443	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPageWithOffset));
1444	# endif
1445
1446	/*
1447	* TlbDone:
1448	*/
1449	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1450	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1451	}
1452	#else
1453	RT_NOREF(idxLabelTlbMiss);
1454	#endif
1455
1456	/*
1457	* Now check the physical address of the page matches the expected one.
1458	*/
1459	RTGCPHYS const GCPhysNewPage = iemTbGetRangePhysPageAddr(pTb, idxRange);
1460	off = iemNativeEmitTestIfGprNotEqualImmAndTbExit(pReNative, off, idxRegGCPhys, GCPhysNewPage,
1461	kIemNativeLabelType_ObsoleteTb);
1462
1463	iemNativeRegFreeTmp(pReNative, idxRegGCPhys);
1464	return off;
1465	}
1466
1467
1468	/**
1469	* Macro that implements TLB loading and updating pbInstrBuf updating when
1470	* branching or when crossing a page on an instruction boundrary.
1471	*
1472	* This differs from BODY_LOAD_TLB_FOR_NEW_PAGE in that it will first check if
1473	* it is an inter-page branch and also check the page offset.
1474	*
1475	* This may long jump if we're raising a \#PF, \#GP or similar trouble.
1476	*/
1477	#define BODY_LOAD_TLB_AFTER_BRANCH(a_pTb, a_idxRange, a_cbInstr) \
1478	RT_NOREF(a_cbInstr); \
1479	off = iemNativeEmitBltLoadTlbAfterBranch(pReNative, off, pTb, a_idxRange)
1480
1481	#define LIVENESS_LOAD_TLB_AFTER_BRANCH(a_pOutgoing, a_pCallEntry) \
1482	if (!IEM_F_MODE_X86_IS_FLAT((uint32_t)(a_pCallEntry)->auParams[0] >> 8)) \
1483	IEM_LIVENESS_RAW_SEG_BASE_INPUT(a_pOutgoing, X86_SREG_CS); \
1484	else do { } while (0)
1485
1486	DECL_FORCE_INLINE(uint32_t)
1487	iemNativeEmitBltLoadTlbAfterBranch(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTB pTb, uint8_t idxRange)
1488	{
1489	#ifdef VBOX_STRICT
1490	off = iemNativeEmitMarker(pReNative, off, 0x80000006);
1491	#endif
1492
1493	BODY_FLUSH_PENDING_WRITES();
1494
1495	/*
1496	* Define labels and allocate the register for holding the GCPhys of the new page.
1497	*/
1498	uint16_t const uTlbSeqNo = pReNative->uTlbSeqNo++;
1499	RTGCPHYS const GCPhysRangePageWithOffset = iemTbGetRangePhysPageAddr(pTb, idxRange)
1500	\| pTb->aRanges[idxRange].offPhysPage;
1501
1502	/*
1503	*
1504	* First check if RIP is within the current code.
1505	*
1506	* This is very similar to iemNativeEmitBltInCheckPcAfterBranch, the only
1507	* difference is what we do when stuff doesn't match up.
1508	*
1509	* What we to do is.
1510	* 1. Calculate the FLAT PC (RIP + CS.BASE).
1511	* 2. Subtract iem.s.uInstrBufPc from it and getting 'off'.
1512	* 3. The 'off' must be less than X86_PAGE_SIZE/cbInstrBufTotal or
1513	* we need to retranslate RIP via the TLB.
1514	* 4. Add 'off' to iem.s.GCPhysInstrBuf and compare with the
1515	* GCPhysRangePageWithOffset constant mentioned above.
1516	*
1517	* The adding of CS.BASE to RIP can be skipped in the first step if we're
1518	* in 64-bit code or flat 32-bit.
1519	*
1520	*/
1521
1522	/* Allocate registers for step 1. Get the shadowed stuff before allocating
1523	the temp register, so we don't accidentally clobber something we'll be
1524	needing again immediately. This is why we get idxRegCsBase here.
1525	Update: We share registers with the TlbState, as the TLB code path has
1526	little in common with the rest of the code. */
1527	bool const fIsFlat = IEM_F_MODE_X86_IS_FLAT(pReNative->fExec);
1528	IEMNATIVEEMITTLBSTATE const TlbState(pReNative, fIsFlat, &off);
1529	uint8_t const idxRegPc = !TlbState.fSkip ? TlbState.idxRegPtr
1530	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, kIemNativeGstReg_Pc,
1531	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1532	uint8_t const idxRegCsBase = !TlbState.fSkip \|\| fIsFlat ? TlbState.idxRegSegBase
1533	: iemNativeRegAllocTmpForGuestReg(pReNative, &off, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS),
1534	kIemNativeGstRegUse_ReadOnly, true /fNoVolatileRegs/);
1535
1536	uint8_t const idxRegTmp = !TlbState.fSkip ? TlbState.idxReg1 : iemNativeRegAllocTmp(pReNative, &off);
1537	uint8_t const idxRegTmp2 = !TlbState.fSkip ? TlbState.idxReg2 : iemNativeRegAllocTmp(pReNative, &off);
1538	uint8_t const idxRegDummy = !TlbState.fSkip ? iemNativeRegAllocTmp(pReNative, &off) : UINT8_MAX;
1539
1540	#ifdef VBOX_STRICT
1541	/* Do assertions before idxRegTmp contains anything. */
1542	Assert(RT_SIZEOFMEMB(VMCPUCC, iem.s.cbInstrBufTotal) == sizeof(uint16_t));
1543	# ifdef RT_ARCH_AMD64
1544	{
1545	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8+2+1 + 11+2+1);
1546	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1547	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1548	{
1549	/* cmp r/m64, imm8 */
1550	pbCodeBuf[off++] = X86_OP_REX_W;
1551	pbCodeBuf[off++] = 0x83;
1552	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 7, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1553	pbCodeBuf[off++] = 0;
1554	/* je rel8 */
1555	pbCodeBuf[off++] = 0x74;
1556	pbCodeBuf[off++] = 1;
1557	/* int3 */
1558	pbCodeBuf[off++] = 0xcc;
1559
1560	}
1561
1562	/* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); - done later by the non-x86 code */
1563	/* test r/m64, imm32 */
1564	pbCodeBuf[off++] = X86_OP_REX_W;
1565	pbCodeBuf[off++] = 0xf7;
1566	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, 0, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1567	pbCodeBuf[off++] = RT_BYTE1(X86_PAGE_OFFSET_MASK);
1568	pbCodeBuf[off++] = RT_BYTE2(X86_PAGE_OFFSET_MASK);
1569	pbCodeBuf[off++] = RT_BYTE3(X86_PAGE_OFFSET_MASK);
1570	pbCodeBuf[off++] = RT_BYTE4(X86_PAGE_OFFSET_MASK);
1571	/* jz rel8 */
1572	pbCodeBuf[off++] = 0x74;
1573	pbCodeBuf[off++] = 1;
1574	/* int3 */
1575	pbCodeBuf[off++] = 0xcc;
1576	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1577	}
1578	# else
1579
1580	/* Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 \|\| !IEM_F_MODE_X86_IS_FLAT(pReNative->fExec)); */
1581	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1582	{
1583	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, cpum.GstCtx.cs.u64Base));
1584	# ifdef RT_ARCH_ARM64
1585	uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1586	pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /fJmpIfNotZero/, 2, idxRegTmp);
1587	pu32CodeBuf[off++] = Armv8A64MkInstrBrk(0x2006);
1588	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1589	# else
1590	# error "Port me!"
1591	# endif
1592	}
1593	# endif
1594
1595	#endif /* VBOX_STRICT */
1596
1597	/* Because we're lazy, we'll jump back here to recalc 'off' and share the
1598	GCPhysRangePageWithOffset check. This is a little risky, so we use the
1599	2nd register to check if we've looped more than once already.*/
1600	off = iemNativeEmitGprZero(pReNative, off, idxRegTmp2);
1601
1602	uint32_t const offLabelRedoChecks = off;
1603
1604	/* 1+2. Calculate 'off' first (into idxRegTmp). */
1605	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc));
1606	if (IEM_F_MODE_X86_IS_FLAT(pReNative->fExec))
1607	{
1608	#ifdef RT_ARCH_ARM64
1609	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1610	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegPc, idxRegTmp);
1611	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1612	#else
1613	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1614	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1615	#endif
1616	}
1617	else
1618	{
1619	#ifdef RT_ARCH_ARM64
1620	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);
1621	pu32CodeBuf[off++] = Armv8A64MkInstrSubReg(idxRegTmp, idxRegCsBase, idxRegTmp);
1622	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegPc);
1623	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1624	#else
1625	off = iemNativeEmitNegGpr(pReNative, off, idxRegTmp);
1626	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegCsBase);
1627	off = iemNativeEmitAddTwoGprs(pReNative, off, idxRegTmp, idxRegPc);
1628	#endif
1629	}
1630
1631	/* 3. Check that off is less than X86_PAGE_SIZE/cbInstrBufTotal.
1632	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll jump to the TLB loading if this fails. */
1633	off = iemNativeEmitCmpGprWithImm(pReNative, off, idxRegTmp, X86_PAGE_SIZE - 1);
1634	uint32_t const offFixedJumpToTlbLoad = off;
1635	off = iemNativeEmitJaToFixed(pReNative, off, off /* (ASSUME ja rel8 suffices) */);
1636
1637	/* 4a. Add iem.s.GCPhysInstrBuf to off ... */
1638	#ifdef RT_ARCH_AMD64
1639	uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1640	pbCodeBuf[off++] = idxRegTmp < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
1641	pbCodeBuf[off++] = 0x03; /* add r64, r/m64 */
1642	off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, idxRegTmp, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1643	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1644
1645	#elif defined(RT_ARCH_ARM64)
1646
1647	off = iemNativeEmitLoadGprFromVCpuU64(pReNative, off, idxRegTmp2, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf));
1648	uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1649	pu32CodeBuf[off++] = Armv8A64MkInstrAddReg(idxRegTmp, idxRegTmp, idxRegTmp2);
1650	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1651
1652	# ifdef VBOX_STRICT /* Assert(!(pVCpu->iem.s.GCPhysInstrBuf & X86_PAGE_OFFSET_MASK)); */
1653	off = iemNativeEmitAndGpr32ByImm(pReNative, off, idxRegTmp2, X86_PAGE_OFFSET_MASK, true /fSetFlags/);
1654	off = iemNativeEmitJzToFixed(pReNative, off, off + 2 /* correct for ARM64 */);
1655	off = iemNativeEmitBrk(pReNative, off, 0x2005);
1656	# endif
1657	#else
1658	# error "Port me"
1659	#endif
1660
1661	/* 4b. ... and compare with GCPhysRangePageWithOffset.
1662
1663	Unlike iemNativeEmitBltInCheckPcAfterBranch we'll have to be more
1664	careful and avoid implicit temporary register usage here.
1665
1666	Unlike the threaded version of this code, we do not obsolete TBs here to
1667	reduce the code size and because indirect calls may legally end at the
1668	same offset in two different pages depending on the program state. */
1669	/** @todo synch the threaded BODY_LOAD_TLB_AFTER_BRANCH version with this. */
1670	off = iemNativeEmitLoadGprImm64(pReNative, off, idxRegTmp2, GCPhysRangePageWithOffset);
1671	off = iemNativeEmitCmpGprWithGpr(pReNative, off, idxRegTmp, idxRegTmp2);
1672	off = iemNativeEmitJnzTbExit(pReNative, off, kIemNativeLabelType_CheckBranchMiss);
1673	uint32_t const offFixedJumpToEnd = off;
1674	off = iemNativeEmitJmpToFixed(pReNative, off, off + 512 /* force rel32 */);
1675
1676	/*
1677	* TlbLoad:
1678	*
1679	* First we try to go via the TLB.
1680	*/
1681	iemNativeFixupFixedJump(pReNative, offFixedJumpToTlbLoad, off);
1682
1683	/* Check that we haven't been here before. */
1684	off = iemNativeEmitTestIfGprIsNotZeroAndTbExit(pReNative, off, idxRegTmp2, false /f64Bit/,
1685	kIemNativeLabelType_CheckBranchMiss);
1686
1687	/* Jump to the TLB lookup code. */
1688	uint32_t const idxLabelTlbLookup = !TlbState.fSkip
1689	? iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbLookup, UINT32_MAX, uTlbSeqNo)
1690	: UINT32_MAX;
1691	//off = iemNativeEmitBrk(pReNative, off, 0x1234);
1692	if (!TlbState.fSkip)
1693	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbLookup); /** @todo short jump */
1694
1695	/*
1696	* TlbMiss:
1697	*
1698	* Call iemNativeHlpMemCodeNewPageTlbMiss to do the work.
1699	*/
1700	uint32_t const idxLabelTlbMiss = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbMiss, off, uTlbSeqNo);
1701	RT_NOREF(idxLabelTlbMiss);
1702
1703	/* Save variables in volatile registers. */
1704	uint32_t const fHstRegsNotToSave = TlbState.getRegsNotToSave() \| RT_BIT_32(idxRegTmp) \| RT_BIT_32(idxRegTmp2)
1705	\| (idxRegDummy != UINT8_MAX ? RT_BIT_32(idxRegDummy) : 0);
1706	off = iemNativeVarSaveVolatileRegsPreHlpCall(pReNative, off, fHstRegsNotToSave);
1707
1708	/* IEMNATIVE_CALL_ARG0_GREG = pVCpu */
1709	off = iemNativeEmitLoadGprFromGpr(pReNative, off, IEMNATIVE_CALL_ARG0_GREG, IEMNATIVE_REG_FIXED_PVMCPU);
1710
1711	/* Done setting up parameters, make the call. */
1712	off = iemNativeEmitCallImm(pReNative, off, (uintptr_t)iemNativeHlpMemCodeNewPageTlbMiss);
1713
1714	/* Restore variables and guest shadow registers to volatile registers. */
1715	off = iemNativeVarRestoreVolatileRegsPostHlpCall(pReNative, off, fHstRegsNotToSave);
1716	off = iemNativeRegRestoreGuestShadowsInVolatileRegs(pReNative, off,
1717	TlbState.getActiveRegsWithShadows()
1718	\| RT_BIT_32(idxRegPc)
1719	\| (idxRegCsBase != UINT8_MAX ? RT_BIT_32(idxRegCsBase) : 0));
1720
1721	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
1722	if (!TlbState.fSkip)
1723	{
1724	/* end of TlbMiss - Jump to the done label. */
1725	uint32_t const idxLabelTlbDone = iemNativeLabelCreate(pReNative, kIemNativeLabelType_TlbDone, UINT32_MAX, uTlbSeqNo);
1726	off = iemNativeEmitJmpToLabel(pReNative, off, idxLabelTlbDone);
1727
1728	/*
1729	* TlbLookup:
1730	*/
1731	off = iemNativeEmitTlbLookup<false, true>(pReNative, off, &TlbState, fIsFlat ? UINT8_MAX : X86_SREG_CS,
1732	1 /cbMem/, 0 /fAlignMask/, IEM_ACCESS_TYPE_EXEC,
1733	idxLabelTlbLookup, idxLabelTlbMiss, idxRegDummy);
1734
1735	# ifdef IEM_WITH_TLB_STATISTICS
1736	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, TlbState.idxReg1, TlbState.idxReg2,
1737	RT_UOFFSETOF(VMCPUCC, iem.s.StatNativeCodeTlbHitsForNewPage));
1738	# endif
1739
1740	/*
1741	* TlbDone:
1742	*/
1743	iemNativeLabelDefine(pReNative, idxLabelTlbDone, off);
1744	TlbState.freeRegsAndReleaseVars(pReNative, UINT8_MAX /idxVarGCPtrMem/, true /fIsCode/);
1745	}
1746	#else
1747	RT_NOREF(idxLabelTlbMiss);
1748	#endif
1749
1750	/* Jmp back to the start and redo the checks. */
1751	off = iemNativeEmitLoadGpr8Imm(pReNative, off, idxRegTmp2, 1); /* indicate that we've looped once already */
1752	off = iemNativeEmitJmpToFixed(pReNative, off, offLabelRedoChecks);
1753
1754	/*
1755	* End:
1756	*
1757	* The end.
1758	*/
1759	iemNativeFixupFixedJump(pReNative, offFixedJumpToEnd, off);
1760
1761	if (!TlbState.fSkip)
1762	iemNativeRegFreeTmp(pReNative, idxRegDummy);
1763	else
1764	{
1765	iemNativeRegFreeTmp(pReNative, idxRegTmp2);
1766	iemNativeRegFreeTmp(pReNative, idxRegTmp);
1767	iemNativeRegFreeTmp(pReNative, idxRegPc);
1768	if (idxRegCsBase != UINT8_MAX)
1769	iemNativeRegFreeTmp(pReNative, idxRegCsBase);
1770	}
1771	return off;
1772	}
1773
1774
1775	#ifdef BODY_CHECK_CS_LIM
1776	/**
1777	* Built-in function that checks the EIP/IP + uParam0 is within CS.LIM,
1778	* raising a \#GP(0) if this isn't the case.
1779	*/
1780	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLim)
1781	{
1782	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1783	BODY_SET_CUR_INSTR();
1784	BODY_FLUSH_PENDING_WRITES();
1785	BODY_CHECK_CS_LIM(cbInstr);
1786	return off;
1787	}
1788
1789	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLim)
1790	{
1791	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1792	LIVENESS_CHECK_CS_LIM(pOutgoing);
1793	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1794	RT_NOREF(pCallEntry);
1795	}
1796	#endif
1797
1798
1799	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_CS_LIM)
1800	/**
1801	* Built-in function for re-checking opcodes and CS.LIM after an instruction
1802	* that may have modified them.
1803	*/
1804	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodes)
1805	{
1806	PCIEMTB const pTb = pReNative->pTbOrg;
1807	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1808	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1809	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1810	BODY_SET_CUR_INSTR();
1811	BODY_FLUSH_PENDING_WRITES();
1812	BODY_CHECK_CS_LIM(cbInstr);
1813	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1814	return off;
1815	}
1816
1817	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodes)
1818	{
1819	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1820	LIVENESS_CHECK_CS_LIM(pOutgoing);
1821	LIVENESS_CHECK_OPCODES(pOutgoing);
1822	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1823	RT_NOREF(pCallEntry);
1824	}
1825	#endif
1826
1827
1828	#if defined(BODY_CHECK_OPCODES)
1829	/**
1830	* Built-in function for re-checking opcodes after an instruction that may have
1831	* modified them.
1832	*/
1833	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodes)
1834	{
1835	PCIEMTB const pTb = pReNative->pTbOrg;
1836	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1837	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1838	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1839	BODY_SET_CUR_INSTR();
1840	BODY_FLUSH_PENDING_WRITES();
1841	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1842	return off;
1843	}
1844
1845	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodes)
1846	{
1847	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1848	LIVENESS_CHECK_OPCODES(pOutgoing);
1849	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1850	RT_NOREF(pCallEntry);
1851	}
1852	#endif
1853
1854
1855	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1856	/**
1857	* Built-in function for re-checking opcodes and considering the need for CS.LIM
1858	* checking after an instruction that may have modified them.
1859	*/
1860	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesConsiderCsLim)
1861	{
1862	PCIEMTB const pTb = pReNative->pTbOrg;
1863	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1864	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1865	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1866	BODY_SET_CUR_INSTR();
1867	BODY_FLUSH_PENDING_WRITES();
1868	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1869	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1870	return off;
1871	}
1872
1873	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesConsiderCsLim)
1874	{
1875	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1876	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1877	LIVENESS_CHECK_OPCODES(pOutgoing);
1878	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1879	RT_NOREF(pCallEntry);
1880	}
1881	#endif
1882
1883
1884	/*
1885	* Post-branching checkers.
1886	*/
1887
1888	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1889	/**
1890	* Built-in function for checking CS.LIM, checking the PC and checking opcodes
1891	* after conditional branching within the same page.
1892	*
1893	* @see iemThreadedFunc_BltIn_CheckPcAndOpcodes
1894	*/
1895	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1896	{
1897	PCIEMTB const pTb = pReNative->pTbOrg;
1898	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1899	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1900	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1901	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1902	BODY_SET_CUR_INSTR();
1903	BODY_FLUSH_PENDING_WRITES();
1904	BODY_CHECK_CS_LIM(cbInstr);
1905	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1906	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1907	//LogFunc(("okay\n"));
1908	return off;
1909	}
1910
1911	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndPcAndOpcodes)
1912	{
1913	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1914	LIVENESS_CHECK_CS_LIM(pOutgoing);
1915	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1916	LIVENESS_CHECK_OPCODES(pOutgoing);
1917	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1918	RT_NOREF(pCallEntry);
1919	}
1920	#endif
1921
1922
1923	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH)
1924	/**
1925	* Built-in function for checking the PC and checking opcodes after conditional
1926	* branching within the same page.
1927	*
1928	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1929	*/
1930	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodes)
1931	{
1932	PCIEMTB const pTb = pReNative->pTbOrg;
1933	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1934	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1935	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1936	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1937	BODY_SET_CUR_INSTR();
1938	BODY_FLUSH_PENDING_WRITES();
1939	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1940	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1941	//LogFunc(("okay\n"));
1942	return off;
1943	}
1944
1945	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodes)
1946	{
1947	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1948	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1949	LIVENESS_CHECK_OPCODES(pOutgoing);
1950	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1951	RT_NOREF(pCallEntry);
1952	}
1953	#endif
1954
1955
1956	#if defined(BODY_CHECK_OPCODES) && defined(BODY_CHECK_PC_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
1957	/**
1958	* Built-in function for checking the PC and checking opcodes and considering
1959	* the need for CS.LIM checking after conditional branching within the same
1960	* page.
1961	*
1962	* @see iemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1963	*/
1964	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1965	{
1966	PCIEMTB const pTb = pReNative->pTbOrg;
1967	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
1968	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
1969	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
1970	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
1971	BODY_SET_CUR_INSTR();
1972	BODY_FLUSH_PENDING_WRITES();
1973	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
1974	BODY_CHECK_PC_AFTER_BRANCH(pTb, idxRange, offRange, cbInstr);
1975	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
1976	//LogFunc(("okay\n"));
1977	return off;
1978	}
1979
1980	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckPcAndOpcodesConsiderCsLim)
1981	{
1982	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1983	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
1984	LIVENESS_CHECK_PC_AFTER_BRANCH(pOutgoing, pCallEntry);
1985	LIVENESS_CHECK_OPCODES(pOutgoing);
1986	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
1987	RT_NOREF(pCallEntry);
1988	}
1989	#endif
1990
1991
1992	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CHECK_CS_LIM)
1993	/**
1994	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
1995	* transitioning to a different code page.
1996	*
1997	* The code page transition can either be natural over onto the next page (with
1998	* the instruction starting at page offset zero) or by means of branching.
1999	*
2000	* @see iemThreadedFunc_BltIn_CheckOpcodesLoadingTlb
2001	*/
2002	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
2003	{
2004	PCIEMTB const pTb = pReNative->pTbOrg;
2005	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2006	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2007	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2008	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2009	BODY_SET_CUR_INSTR();
2010	BODY_FLUSH_PENDING_WRITES();
2011	BODY_CHECK_CS_LIM(cbInstr);
2012	Assert(offRange == 0);
2013	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
2014	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2015	//LogFunc(("okay\n"));
2016	return off;
2017	}
2018
2019	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb)
2020	{
2021	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2022	LIVENESS_CHECK_CS_LIM(pOutgoing);
2023	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
2024	LIVENESS_CHECK_OPCODES(pOutgoing);
2025	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2026	RT_NOREF(pCallEntry);
2027	}
2028	#endif
2029
2030
2031	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH)
2032	/**
2033	* Built-in function for loading TLB and checking opcodes when transitioning to
2034	* a different code page.
2035	*
2036	* The code page transition can either be natural over onto the next page (with
2037	* the instruction starting at page offset zero) or by means of branching.
2038	*
2039	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
2040	*/
2041	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlb)
2042	{
2043	PCIEMTB const pTb = pReNative->pTbOrg;
2044	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2045	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2046	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2047	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2048	BODY_SET_CUR_INSTR();
2049	BODY_FLUSH_PENDING_WRITES();
2050	Assert(offRange == 0);
2051	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
2052	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2053	//LogFunc(("okay\n"));
2054	return off;
2055	}
2056
2057	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlb)
2058	{
2059	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2060	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
2061	LIVENESS_CHECK_OPCODES(pOutgoing);
2062	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2063	RT_NOREF(pCallEntry);
2064	}
2065	#endif
2066
2067
2068	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_AFTER_BRANCH) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2069	/**
2070	* Built-in function for loading TLB and checking opcodes and considering the
2071	* need for CS.LIM checking when transitioning to a different code page.
2072	*
2073	* The code page transition can either be natural over onto the next page (with
2074	* the instruction starting at page offset zero) or by means of branching.
2075	*
2076	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
2077	*/
2078	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
2079	{
2080	PCIEMTB const pTb = pReNative->pTbOrg;
2081	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2082	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2083	uint32_t const offRange = (uint32_t)pCallEntry->auParams[2];
2084	//LogFunc(("idxRange=%u @ %#x LB %#x: offPhysPage=%#x LB %#x\n", idxRange, offRange, cbInstr, pTb->aRanges[idxRange].offPhysPage, pTb->aRanges[idxRange].cbOpcodes));
2085	BODY_SET_CUR_INSTR();
2086	BODY_FLUSH_PENDING_WRITES();
2087	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2088	Assert(offRange == 0);
2089	BODY_LOAD_TLB_AFTER_BRANCH(pTb, idxRange, cbInstr);
2090	BODY_CHECK_OPCODES(pTb, idxRange, offRange, cbInstr);
2091	//LogFunc(("okay\n"));
2092	return off;
2093	}
2094
2095	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesLoadingTlbConsiderCsLim)
2096	{
2097	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2098	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2099	LIVENESS_LOAD_TLB_AFTER_BRANCH(pOutgoing, pCallEntry);
2100	LIVENESS_CHECK_OPCODES(pOutgoing);
2101	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2102	RT_NOREF(pCallEntry);
2103	}
2104	#endif
2105
2106
2107
2108	/*
2109	* Natural page crossing checkers.
2110	*/
2111
2112	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2113	/**
2114	* Built-in function for checking CS.LIM, loading TLB and checking opcodes on
2115	* both pages when transitioning to a different code page.
2116	*
2117	* This is used when the previous instruction requires revalidation of opcodes
2118	* bytes and the current instruction stries a page boundrary with opcode bytes
2119	* in both the old and new page.
2120	*
2121	* @see iemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb
2122	*/
2123	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2124	{
2125	PCIEMTB const pTb = pReNative->pTbOrg;
2126	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2127	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2128	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2129	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2130	uint32_t const idxRange2 = idxRange1 + 1;
2131	BODY_SET_CUR_INSTR();
2132	BODY_FLUSH_PENDING_WRITES();
2133	BODY_CHECK_CS_LIM(cbInstr);
2134	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2135	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2136	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2137	return off;
2138	}
2139
2140	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb)
2141	{
2142	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2143	LIVENESS_CHECK_CS_LIM(pOutgoing);
2144	LIVENESS_CHECK_OPCODES(pOutgoing);
2145	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2146	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2147	RT_NOREF(pCallEntry);
2148	}
2149	#endif
2150
2151
2152	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2153	/**
2154	* Built-in function for loading TLB and checking opcodes on both pages when
2155	* transitioning to a different code page.
2156	*
2157	* This is used when the previous instruction requires revalidation of opcodes
2158	* bytes and the current instruction stries a page boundrary with opcode bytes
2159	* in both the old and new page.
2160	*
2161	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2162	*/
2163	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2164	{
2165	PCIEMTB const pTb = pReNative->pTbOrg;
2166	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2167	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2168	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2169	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2170	uint32_t const idxRange2 = idxRange1 + 1;
2171	BODY_SET_CUR_INSTR();
2172	BODY_FLUSH_PENDING_WRITES();
2173	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2174	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2175	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2176	return off;
2177	}
2178
2179	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb)
2180	{
2181	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2182	LIVENESS_CHECK_OPCODES(pOutgoing);
2183	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2184	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2185	RT_NOREF(pCallEntry);
2186	}
2187	#endif
2188
2189
2190	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2191	/**
2192	* Built-in function for loading TLB and checking opcodes on both pages and
2193	* considering the need for CS.LIM checking when transitioning to a different
2194	* code page.
2195	*
2196	* This is used when the previous instruction requires revalidation of opcodes
2197	* bytes and the current instruction stries a page boundrary with opcode bytes
2198	* in both the old and new page.
2199	*
2200	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
2201	*/
2202	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2203	{
2204	PCIEMTB const pTb = pReNative->pTbOrg;
2205	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2206	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2207	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2208	uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2209	uint32_t const idxRange2 = idxRange1 + 1;
2210	BODY_SET_CUR_INSTR();
2211	BODY_FLUSH_PENDING_WRITES();
2212	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2213	BODY_CHECK_OPCODES(pTb, idxRange1, offRange1, cbInstr);
2214	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2215	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2216	return off;
2217	}
2218
2219	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesAcrossPageLoadingTlbConsiderCsLim)
2220	{
2221	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2222	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2223	LIVENESS_CHECK_OPCODES(pOutgoing);
2224	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2225	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2226	RT_NOREF(pCallEntry);
2227	}
2228	#endif
2229
2230
2231	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2232	/**
2233	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2234	* advancing naturally to a different code page.
2235	*
2236	* Only opcodes on the new page is checked.
2237	*
2238	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb
2239	*/
2240	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2241	{
2242	PCIEMTB const pTb = pReNative->pTbOrg;
2243	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2244	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2245	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2246	//uint32_t const offRange1 = (uint32_t)uParam2;
2247	uint32_t const idxRange2 = idxRange1 + 1;
2248	BODY_SET_CUR_INSTR();
2249	BODY_FLUSH_PENDING_WRITES();
2250	BODY_CHECK_CS_LIM(cbInstr);
2251	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2252	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2253	return off;
2254	}
2255
2256	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb)
2257	{
2258	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2259	LIVENESS_CHECK_CS_LIM(pOutgoing);
2260	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2261	LIVENESS_CHECK_OPCODES(pOutgoing);
2262	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2263	RT_NOREF(pCallEntry);
2264	}
2265	#endif
2266
2267
2268	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2269	/**
2270	* Built-in function for loading TLB and checking opcodes when advancing
2271	* naturally to a different code page.
2272	*
2273	* Only opcodes on the new page is checked.
2274	*
2275	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2276	*/
2277	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2278	{
2279	PCIEMTB const pTb = pReNative->pTbOrg;
2280	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2281	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2282	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2283	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2284	uint32_t const idxRange2 = idxRange1 + 1;
2285	BODY_SET_CUR_INSTR();
2286	BODY_FLUSH_PENDING_WRITES();
2287	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2288	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2289	return off;
2290	}
2291
2292	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb)
2293	{
2294	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2295	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2296	LIVENESS_CHECK_OPCODES(pOutgoing);
2297	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2298	RT_NOREF(pCallEntry);
2299	}
2300	#endif
2301
2302
2303	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2304	/**
2305	* Built-in function for loading TLB and checking opcodes and considering the
2306	* need for CS.LIM checking when advancing naturally to a different code page.
2307	*
2308	* Only opcodes on the new page is checked.
2309	*
2310	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
2311	*/
2312	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2313	{
2314	PCIEMTB const pTb = pReNative->pTbOrg;
2315	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2316	uint32_t const cbStartPage = (uint32_t)(pCallEntry->auParams[0] >> 32);
2317	uint32_t const idxRange1 = (uint32_t)pCallEntry->auParams[1];
2318	//uint32_t const offRange1 = (uint32_t)pCallEntry->auParams[2];
2319	uint32_t const idxRange2 = idxRange1 + 1;
2320	BODY_SET_CUR_INSTR();
2321	BODY_FLUSH_PENDING_WRITES();
2322	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2323	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, cbStartPage, idxRange2, cbInstr);
2324	BODY_CHECK_OPCODES(pTb, idxRange2, 0, cbInstr);
2325	return off;
2326	}
2327
2328	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNextPageLoadingTlbConsiderCsLim)
2329	{
2330	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2331	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2332	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2333	LIVENESS_CHECK_OPCODES(pOutgoing);
2334	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2335	RT_NOREF(pCallEntry);
2336	}
2337	#endif
2338
2339
2340	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CHECK_CS_LIM)
2341	/**
2342	* Built-in function for checking CS.LIM, loading TLB and checking opcodes when
2343	* advancing naturally to a different code page with first instr at byte 0.
2344	*
2345	* @see iemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb
2346	*/
2347	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2348	{
2349	PCIEMTB const pTb = pReNative->pTbOrg;
2350	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2351	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2352	BODY_SET_CUR_INSTR();
2353	BODY_FLUSH_PENDING_WRITES();
2354	BODY_CHECK_CS_LIM(cbInstr);
2355	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2356	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2357	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2358	return off;
2359	}
2360
2361	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb)
2362	{
2363	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2364	LIVENESS_CHECK_CS_LIM(pOutgoing);
2365	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2366	LIVENESS_CHECK_OPCODES(pOutgoing);
2367	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2368	RT_NOREF(pCallEntry);
2369	}
2370	#endif
2371
2372
2373	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE)
2374	/**
2375	* Built-in function for loading TLB and checking opcodes when advancing
2376	* naturally to a different code page with first instr at byte 0.
2377	*
2378	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2379	*/
2380	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2381	{
2382	PCIEMTB const pTb = pReNative->pTbOrg;
2383	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2384	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2385	BODY_SET_CUR_INSTR();
2386	BODY_FLUSH_PENDING_WRITES();
2387	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2388	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2389	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2390	return off;
2391	}
2392
2393	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb)
2394	{
2395	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2396	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2397	LIVENESS_CHECK_OPCODES(pOutgoing);
2398	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2399	RT_NOREF(pCallEntry);
2400	}
2401	#endif
2402
2403
2404	#if defined(BODY_CHECK_OPCODES) && defined(BODY_LOAD_TLB_FOR_NEW_PAGE) && defined(BODY_CONSIDER_CS_LIM_CHECKING)
2405	/**
2406	* Built-in function for loading TLB and checking opcodes and considering the
2407	* need for CS.LIM checking when advancing naturally to a different code page
2408	* with first instr at byte 0.
2409	*
2410	* @see iemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
2411	*/
2412	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2413	{
2414	PCIEMTB const pTb = pReNative->pTbOrg;
2415	uint32_t const cbInstr = (uint8_t)pCallEntry->auParams[0];
2416	uint32_t const idxRange = (uint32_t)pCallEntry->auParams[1];
2417	BODY_SET_CUR_INSTR();
2418	BODY_FLUSH_PENDING_WRITES();
2419	BODY_CONSIDER_CS_LIM_CHECKING(pTb, cbInstr);
2420	BODY_LOAD_TLB_FOR_NEW_PAGE(pTb, 0, idxRange, cbInstr);
2421	//Assert(pVCpu->iem.s.offCurInstrStart == 0);
2422	BODY_CHECK_OPCODES(pTb, idxRange, 0, cbInstr);
2423	return off;
2424	}
2425
2426	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_CheckOpcodesOnNewPageLoadingTlbConsiderCsLim)
2427	{
2428	IEM_LIVENESS_RAW_INIT_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2429	LIVENESS_CONSIDER_CS_LIM_CHECKING(pOutgoing);
2430	LIVENESS_LOAD_TLB_FOR_NEW_PAGE(pOutgoing, pCallEntry);
2431	LIVENESS_CHECK_OPCODES(pOutgoing);
2432	IEM_LIVENESS_RAW_FINISH_WITH_POTENTIAL_CALL(pOutgoing, pIncoming);
2433	RT_NOREF(pCallEntry);
2434	}
2435	#endif
2436
2437
2438	/**
2439	* Built-in function for jumping in the call sequence.
2440	*/
2441	IEM_DECL_IEMNATIVERECOMPFUNC_DEF(iemNativeRecompFunc_BltIn_Jump)
2442	{
2443	PCIEMTB const pTb = pReNative->pTbOrg;
2444	Assert(pCallEntry->auParams[1] == 0 && pCallEntry->auParams[2] == 0);
2445	Assert(pCallEntry->auParams[0] < pTb->Thrd.cCalls);
2446	#if 1
2447	RT_NOREF(pCallEntry, pTb);
2448
2449	# ifdef VBOX_WITH_STATISTICS
2450	/* Increment StatNativeTbExitLoopFullTb. */
2451	uint32_t const offStat = RT_UOFFSETOF(VMCPU, iem.s.StatNativeTbExitLoopFullTb);
2452	# ifdef RT_ARCH_AMD64
2453	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, UINT8_MAX, UINT8_MAX, offStat);
2454	# else
2455	uint8_t const idxStatsTmp1 = iemNativeRegAllocTmp(pReNative, &off);
2456	uint8_t const idxStatsTmp2 = iemNativeRegAllocTmp(pReNative, &off);
2457	off = iemNativeEmitIncStamCounterInVCpu(pReNative, off, idxStatsTmp1, idxStatsTmp2, offStat);
2458	iemNativeRegFreeTmp(pReNative, idxStatsTmp1);
2459	iemNativeRegFreeTmp(pReNative, idxStatsTmp2);
2460	# endif
2461	# endif
2462	# ifdef IEMNATIVE_WITH_INSTRUCTION_COUNTING
2463	/** @todo
2464	off = iemNativeEmitAddU32CounterInVCpuEx(pReNative, off, pTb->cInstructions, RT_UOFFSETOF(VMCPUCC, iem.s.cInstructions));
2465	*/
2466	# endif
2467
2468	/* Jump to the start of the TB. */
2469	uint32_t idxLabel = iemNativeLabelFind(pReNative, kIemNativeLabelType_LoopJumpTarget);
2470	AssertStmt(idxLabel < pReNative->cLabels, IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_LABEL_IPE_6)); /** @todo better status */
2471	return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);
2472	#else
2473	RT_NOREF(pReNative, pCallEntry, pTb);
2474	return off;
2475	#endif
2476	}
2477
2478	IEM_DECL_IEMNATIVELIVENESSFUNC_DEF(iemNativeLivenessFunc_BltIn_Jump)
2479	{
2480	/* We could also use UNUSED here, but this'll is equivialent (at the moment). */
2481	IEM_LIVENESS_RAW_INIT_WITH_CALL(pOutgoing, pIncoming);
2482	RT_NOREF(pCallEntry);
2483	}
2484

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllN8veRecompBltIn.cpp@ 106125

Download in other formats: