IEMN8veRecompilerTlbLookup.h@ 103265

Last change on this file since 103265 was 102856, checked in by vboxsync, 11 months ago
VMM/IEM: Implemented the second of two code TLB lookups. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 37.9 KB

Line
1	/* $Id: IEMN8veRecompilerTlbLookup.h 102856 2024-01-12 11:45:10Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler TLB Lookup Code Emitter.
4	*/
5
6	/*
7	* Copyright (C) 2023-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35	#include "IEMN8veRecompilerEmit.h"
36
37
38	/** @defgroup grp_iem_n8ve_re_tlblookup Native Recompiler TLB Lookup Code Emitter
39	* @ingroup grp_iem_n8ve_re
40	* @{
41	*/
42
43	/*
44	* TLB Lookup config.
45	*/
46	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
47	# define IEMNATIVE_WITH_TLB_LOOKUP
48	#endif
49	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
50	# define IEMNATIVE_WITH_TLB_LOOKUP_FETCH
51	#endif
52	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
53	# define IEMNATIVE_WITH_TLB_LOOKUP_STORE
54	#endif
55	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
56	# define IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
57	#endif
58	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
59	# define IEMNATIVE_WITH_TLB_LOOKUP_PUSH
60	#endif
61	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
62	# define IEMNATIVE_WITH_TLB_LOOKUP_POP
63	#endif
64
65
66	/**
67	* This must be instantiate before branching off to the lookup code,
68	* so that register spilling and whatnot happens for everyone.
69	*/
70	typedef struct IEMNATIVEEMITTLBSTATE
71	{
72	bool const fSkip;
73	uint8_t const idxRegPtrHlp; /*< We don't support immediate variables with register assignment, so this a tmp reg alloc. /
74	uint8_t const idxRegPtr;
75	uint8_t const idxRegSegBase;
76	uint8_t const idxRegSegLimit;
77	uint8_t const idxRegSegAttrib;
78	uint8_t const idxReg1;
79	uint8_t const idxReg2;
80	#if defined(RT_ARCH_ARM64)
81	uint8_t const idxReg3;
82	#endif
83	uint64_t const uAbsPtr;
84
85	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint32_t *a_poff, uint8_t a_idxVarGCPtrMem,
86	uint8_t a_iSegReg, uint8_t a_cbMem, uint8_t a_offDisp = 0)
87	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
88	/* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
89	: fSkip( a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind == kIemNativeVarKind_Immediate
90	&& ( (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
91	? (uint64_t)(UINT32_MAX - a_cbMem - a_offDisp)
92	: (uint64_t)(UINT64_MAX - a_cbMem - a_offDisp))
93	< a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
94	#else
95	: fSkip(true)
96	#endif
97	#if defined(RT_ARCH_AMD64) /* got good immediate encoding, otherwise we just load the address in a reg immediately. */
98	, idxRegPtrHlp(UINT8_MAX)
99	#else
100	, idxRegPtrHlp( a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate
101	\|\| fSkip
102	? UINT8_MAX
103	: iemNativeRegAllocTmpImm(a_pReNative, a_poff, a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue) )
104	#endif
105	, idxRegPtr(a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate && !fSkip
106	? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
107	true /fInitialized/, IEMNATIVE_CALL_ARG2_GREG)
108	: idxRegPtrHlp)
109	, idxRegSegBase(a_iSegReg == UINT8_MAX \|\| fSkip
110	? UINT8_MAX
111	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
112	, idxRegSegLimit((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
113	? UINT8_MAX
114	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
115	, idxRegSegAttrib((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
116	? UINT8_MAX
117	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
118	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
119	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
120	#if defined(RT_ARCH_ARM64)
121	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
122	#endif
123	, uAbsPtr( a_pReNative->Core.aVars[a_idxVarGCPtrMem].enmKind != kIemNativeVarKind_Immediate \|\| fSkip
124	? UINT64_MAX
125	: a_pReNative->Core.aVars[a_idxVarGCPtrMem].u.uValue)
126
127	{
128	RT_NOREF(a_cbMem, a_offDisp);
129	}
130
131	/* Alternative constructor for PUSH and POP where we don't have a GCPtrMem
132	variable, only a register derived from the guest RSP. */
133	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint8_t a_idxRegPtr, uint32_t *a_poff,
134	uint8_t a_iSegReg, uint8_t a_cbMem)
135	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
136	: fSkip(false)
137	#else
138	: fSkip(true)
139	#endif
140	, idxRegPtrHlp(UINT8_MAX)
141	, idxRegPtr(a_idxRegPtr)
142	, idxRegSegBase(a_iSegReg == UINT8_MAX \|\| fSkip
143	? UINT8_MAX
144	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
145	, idxRegSegLimit((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
146	? UINT8_MAX
147	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
148	, idxRegSegAttrib((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
149	? UINT8_MAX
150	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
151	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
152	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
153	#if defined(RT_ARCH_ARM64)
154	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
155	#endif
156	, uAbsPtr(UINT64_MAX)
157
158	{
159	RT_NOREF_PV(a_cbMem);
160	}
161
162	/* Alternative constructor for the code TLB lookups where we implictly use RIP
163	variable, only a register derived from the guest RSP. */
164	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, bool a_fFlat, uint32_t *a_poff)
165	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
166	: fSkip(false)
167	#else
168	: fSkip(true)
169	#endif
170	, idxRegPtrHlp(UINT8_MAX)
171	, idxRegPtr(iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, kIemNativeGstReg_Pc))
172	, idxRegSegBase(a_fFlat \|\| fSkip
173	? UINT8_MAX
174	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS)))
175	, idxRegSegLimit(/*a_fFlat \|\| fSkip
176	? UINT8_MAX
177	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS))*/
178	UINT8_MAX)
179	, idxRegSegAttrib(UINT8_MAX)
180	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
181	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
182	#if defined(RT_ARCH_ARM64)
183	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
184	#endif
185	, uAbsPtr(UINT64_MAX)
186
187	{
188	}
189
190	void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX, bool fIsCode = false) const
191	{
192	if (!fIsCode)
193	{
194	if (idxRegPtr != UINT8_MAX)
195	{
196	if (idxRegPtrHlp == UINT8_MAX)
197	{
198	if (idxVarGCPtrMem != UINT8_MAX)
199	iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
200	}
201	else
202	{
203	Assert(idxRegPtrHlp == idxRegPtr);
204	iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
205	}
206	}
207	else
208	Assert(idxRegPtrHlp == UINT8_MAX);
209	}
210	else
211	{
212	Assert(idxVarGCPtrMem == UINT8_MAX);
213	Assert(idxRegPtrHlp == UINT8_MAX);
214	iemNativeRegFreeTmp(a_pReNative, idxRegPtr); /* RIP */
215	}
216	if (idxRegSegBase != UINT8_MAX)
217	iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
218	if (idxRegSegLimit != UINT8_MAX)
219	iemNativeRegFreeTmp(a_pReNative, idxRegSegLimit);
220	if (idxRegSegAttrib != UINT8_MAX)
221	iemNativeRegFreeTmp(a_pReNative, idxRegSegAttrib);
222	#if defined(RT_ARCH_ARM64)
223	iemNativeRegFreeTmp(a_pReNative, idxReg3);
224	#endif
225	iemNativeRegFreeTmp(a_pReNative, idxReg2);
226	iemNativeRegFreeTmp(a_pReNative, idxReg1);
227
228	}
229
230	uint32_t getRegsNotToSave() const
231	{
232	if (!fSkip)
233	return RT_BIT_32(idxReg1)
234	\| RT_BIT_32(idxReg2)
235	#if defined(RT_ARCH_ARM64)
236	\| RT_BIT_32(idxReg3)
237	#endif
238	;
239	return 0;
240	}
241
242	/** This is only for avoid assertions. */
243	uint32_t getActiveRegsWithShadows(bool fCode = false) const
244	{
245	#ifdef VBOX_STRICT
246	if (!fSkip)
247	return (idxRegSegBase != UINT8_MAX ? RT_BIT_32(idxRegSegBase) : 0)
248	\| (idxRegSegLimit != UINT8_MAX ? RT_BIT_32(idxRegSegLimit) : 0)
249	\| (idxRegSegAttrib != UINT8_MAX ? RT_BIT_32(idxRegSegAttrib) : 0)
250	\| (fCode ? RT_BIT_32(idxRegPtr) : 0);
251	#else
252	RT_NOREF_PV(fCode);
253	#endif
254	return 0;
255	}
256	} IEMNATIVEEMITTLBSTATE;
257
258	DECLASM(void) iemNativeHlpAsmSafeWrapCheckTlbLookup(void);
259
260
261	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
262	template<bool const a_fDataTlb, bool const a_fNoReturn = false>
263	DECL_INLINE_THROW(uint32_t)
264	iemNativeEmitTlbLookup(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEMITTLBSTATE const * const pTlbState,
265	uint8_t iSegReg, uint8_t cbMem, uint8_t fAlignMask, uint32_t fAccess,
266	uint32_t idxLabelTlbLookup, uint32_t idxLabelTlbMiss, uint8_t idxRegMemResult,
267	uint8_t offDisp = 0)
268	{
269	Assert(!pTlbState->fSkip);
270	uint32_t const offVCpuTlb = a_fDataTlb ? RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb) : RT_UOFFSETOF(VMCPUCC, iem.s.CodeTlb);
271	# if defined(RT_ARCH_AMD64)
272	uint8_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 512);
273	# elif defined(RT_ARCH_ARM64)
274	uint32_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
275	# endif
276
277	/*
278	* The expand down check isn't use all that much, so we emit here to keep
279	* the lookup straighter.
280	*/
281	/* check_expand_down: ; complicted! */
282	uint32_t const offCheckExpandDown = off;
283	uint32_t offFixupLimitDone = 0;
284	if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
285	{
286	off = iemNativeEmitBrkEx(pCodeBuf, off, 1); /** @todo this needs testing */
287	/* cmp seglim, regptr */
288	if (pTlbState->idxRegPtr != UINT8_MAX && offDisp == 0)
289	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxRegPtr);
290	else if (pTlbState->idxRegPtr == UINT8_MAX)
291	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
292	(uint32_t)(pTlbState->uAbsPtr + offDisp));
293	else if (cbMem == 1)
294	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxReg2);
295	else
296	{ /* use idxRegMemResult to calc the displaced address. */
297	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxRegPtr, offDisp);
298	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, idxRegMemResult);
299	}
300	/* ja tlbmiss */
301	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
302
303	/* reg1 = segattr & X86DESCATTR_D (0x4000) */
304	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib, X86DESCATTR_D);
305	/* xor reg1, X86DESCATTR_D */
306	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
307	/* shl reg1, 2 (16 - 14) */
308	AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
309	off = iemNativeEmitShiftGpr32LeftEx(pCodeBuf, off, pTlbState->idxReg1, 2);
310	/* dec reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
311	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, 1);
312	/* cmp reg1, reg2 (64-bit) / imm (32-bit) */
313	if (pTlbState->idxRegPtr != UINT8_MAX)
314	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1,
315	cbMem > 1 \|\| offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
316	else
317	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1,
318	(uint32_t)(pTlbState->uAbsPtr + offDisp + cbMem - 1)); /* fSkip=true on overflow. */
319	/* jbe tlbmiss */
320	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
321	/* jmp limitdone */
322	offFixupLimitDone = off;
323	off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, off /* ASSUME short jump suffices */);
324	}
325
326	/*
327	* tlblookup:
328	*/
329	iemNativeLabelDefine(pReNative, idxLabelTlbLookup, off);
330	# if defined(RT_ARCH_ARM64) && 0
331	off = iemNativeEmitBrkEx(pCodeBuf, off, 0);
332	# endif
333
334	/*
335	* 1. Segmentation.
336	*
337	* 1a. Check segment limit and attributes if non-flat 32-bit code. This is complicated.
338	*
339	* This can be skipped for code TLB lookups because limit is checked by jmp, call,
340	* ret, and iret prior to making it. It is also checked by the helpers prior to
341	* doing TLB loading.
342	*/
343	if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
344	{
345	/* Check that we've got a segment loaded and that it allows the access.
346	For write access this means a writable data segment.
347	For read-only accesses this means a readable code segment or any data segment. */
348	if (fAccess & IEM_ACCESS_TYPE_WRITE)
349	{
350	uint32_t const fMustBe1 = X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_WRITE;
351	uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE \| X86_SEL_TYPE_CODE;
352	/* reg1 = segattrs & (must1\|must0) */
353	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
354	pTlbState->idxRegSegAttrib, fMustBe1 \| fMustBe0);
355	/* cmp reg1, must1 */
356	AssertCompile(fMustBe1 <= UINT16_MAX);
357	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1);
358	/* jne tlbmiss */
359	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
360	}
361	else
362	{
363	/* U \| !P \|!DT \|!CD \| RW \|
364	16 \| 8 \| 4 \| 3 \| 1 \|
365	-------------------------------
366	0 \| 0 \| 0 \| 0 \| 0 \| execute-only code segment. - must be excluded
367	0 \| 0 \| 0 \| 0 \| 1 \| execute-read code segment.
368	0 \| 0 \| 0 \| 1 \| 0 \| read-only data segment.
369	0 \| 0 \| 0 \| 1 \| 1 \| read-write data segment. - last valid combination
370	*/
371	/* reg1 = segattrs & (relevant attributes) */
372	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib,
373	X86DESCATTR_UNUSABLE \| X86DESCATTR_P \| X86DESCATTR_DT
374	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE);
375	/* xor reg1, X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
376	; EO-code=0, ER-code=2, RO-data=8, RW-data=10 */
377	#ifdef RT_ARCH_ARM64
378	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_DT \| X86_SEL_TYPE_CODE);
379	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_P);
380	#else
381	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1,
382	X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_CODE);
383	#endif
384	/* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
385	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
386	/* cmp reg1, X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE */
387	AssertCompile(X86_SEL_TYPE_CODE == 8);
388	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_CODE);
389	/* ja tlbmiss */
390	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
391	}
392
393	/* If we're accessing more than one byte or if we're working with a non-zero offDisp,
394	put the last address we'll be accessing in idxReg2 (64-bit). */
395	if ((cbMem > 1 \|\| offDisp != 0) && pTlbState->idxRegPtr != UINT8_MAX)
396	{
397	if (!offDisp)
398	/* reg2 = regptr + cbMem - 1; 64-bit result so we can fend of wraparounds/overflows. */
399	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, pTlbState->idxReg2,/=/ pTlbState->idxRegPtr,/+/ cbMem - 1);
400	else
401	{
402	/* reg2 = (uint32_t)(regptr + offDisp) + cbMem - 1;. */
403	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off,
404	pTlbState->idxReg2,/=/ pTlbState->idxRegPtr,/+/ + offDisp);
405	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
406	}
407	}
408
409	/*
410	* Check the limit. If this is a write access, we know that it's a
411	* data segment and includes the expand_down bit. For read-only accesses
412	* we need to check that code/data=0 and expanddown=1 before continuing.
413	*/
414	if (fAccess & IEM_ACCESS_TYPE_WRITE)
415	{
416	/* test segattrs, X86_SEL_TYPE_DOWN */
417	AssertCompile(X86_SEL_TYPE_DOWN < 128);
418	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, pTlbState->idxRegSegAttrib, X86_SEL_TYPE_DOWN);
419	/* jnz check_expand_down */
420	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_ne);
421	}
422	else
423	{
424	/* reg1 = segattr & (code \| down) */
425	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
426	pTlbState->idxRegSegAttrib, X86_SEL_TYPE_CODE \| X86_SEL_TYPE_DOWN);
427	/* cmp reg1, down */
428	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_DOWN);
429	/* je check_expand_down */
430	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_e);
431	}
432
433	/* expand_up:
434	cmp seglim, regptr/reg2/imm */
435	if (pTlbState->idxRegPtr != UINT8_MAX)
436	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
437	cbMem > 1 \|\| offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
438	else
439	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
440	(uint32_t)pTlbState->uAbsPtr + offDisp + cbMem - 1U); /* fSkip=true on overflow. */
441	/* jbe tlbmiss */
442	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
443
444	/* limitdone: */
445	iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
446	}
447
448	/* 1b. Add the segment base. We use idxRegMemResult for the ptr register if
449	this step is required or if the address is a constant (simplicity) or
450	if offDisp is non-zero. */
451	uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX \|\| pTlbState->idxRegPtr == UINT8_MAX \|\| offDisp != 0
452	? idxRegMemResult : pTlbState->idxRegPtr;
453	if (iSegReg != UINT8_MAX)
454	{
455	Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
456	/* regflat = segbase + regptr/imm */
457	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
458	{
459	Assert(iSegReg >= X86_SREG_FS);
460	if (pTlbState->idxRegPtr != UINT8_MAX)
461	{
462	off = iemNativeEmitGprEqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
463	if (offDisp != 0)
464	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
465	}
466	else
467	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase,
468	pTlbState->uAbsPtr + offDisp);
469	}
470	else if (pTlbState->idxRegPtr != UINT8_MAX)
471	{
472	off = iemNativeEmitGpr32EqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
473	if (offDisp != 0)
474	off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
475	}
476	else
477	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr,
478	pTlbState->idxRegSegBase, (uint32_t)pTlbState->uAbsPtr + offDisp);
479	}
480	else if (pTlbState->idxRegPtr == UINT8_MAX)
481	{
482	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
483	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr + offDisp);
484	else
485	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, (uint32_t)pTlbState->uAbsPtr + offDisp);
486	}
487	else if (offDisp != 0)
488	{
489	Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
490	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
491	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
492	else
493	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
494	}
495	else
496	Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
497
498	/*
499	* 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
500	*
501	* 2a. Alignment check using fAlignMask.
502	*/
503	if (fAlignMask)
504	{
505	Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
506	Assert(fAlignMask < 128);
507	/* test regflat, fAlignMask */
508	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, idxRegFlatPtr, fAlignMask);
509	/* jnz tlbmiss */
510	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
511	}
512
513	/*
514	* 2b. Check that it's not crossing page a boundrary. This is implicit in
515	* the previous test if the alignment is same or larger than the type.
516	*/
517	if (cbMem > fAlignMask + 1)
518	{
519	/* reg1 = regflat & 0xfff */
520	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,/=/ idxRegFlatPtr,/&/ GUEST_PAGE_OFFSET_MASK);
521	/* cmp reg1, GUEST_PAGE_SIZE - cbMem */
522	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE);
523	/* ja tlbmiss */
524	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
525	}
526
527	/*
528	* 3. TLB lookup.
529	*
530	* 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
531	* In 64-bit mode we will also check for non-canonical addresses here.
532	*/
533	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
534	{
535	# if defined(RT_ARCH_AMD64)
536	/* mov reg1, regflat */
537	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
538	/* rol reg1, 16 */
539	off = iemNativeEmitRotateGprLeftEx(pCodeBuf, off, pTlbState->idxReg1, 16);
540	/** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
541	/* inc word reg1 */
542	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
543	if (pTlbState->idxReg1 >= 8)
544	pCodeBuf[off++] = X86_OP_REX_B;
545	pCodeBuf[off++] = 0xff;
546	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, pTlbState->idxReg1 & 7);
547	/* cmp word reg1, 1 */
548	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
549	if (pTlbState->idxReg1 >= 8)
550	pCodeBuf[off++] = X86_OP_REX_B;
551	pCodeBuf[off++] = 0x83;
552	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, pTlbState->idxReg1 & 7);
553	pCodeBuf[off++] = 1;
554	/* ja tlbmiss */
555	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
556	/* shr reg1, 16 + GUEST_PAGE_SHIFT */
557	off = iemNativeEmitShiftGprRightEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
558
559	# elif defined(RT_ARCH_ARM64)
560	/* lsr reg1, regflat, #48 */
561	pCodeBuf[off++] = Armv8A64MkInstrLslImm(pTlbState->idxReg1, idxRegFlatPtr, 4);
562	/* add reg1, reg1, #1 */
563	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(pTlbState->idxReg1, pTlbState->idxReg1, 1, false /f64Bit/);
564	/* tst reg1, #0xfffe */
565	Assert(Armv8A64ConvertImmRImmS2Mask32(14, 31) == 0xfffe);
566	pCodeBuf[off++] = Armv8A64MkInstrTstImm(pTlbState->idxReg1, 14, 31, false /f64Bit/);
567	/* b.nq tlbmiss */
568	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
569
570	/* ubfx reg1, regflat, #12, #36 */
571	pCodeBuf[off++] = Armv8A64MkInstrUbfx(pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT, 48 - GUEST_PAGE_SHIFT);
572	# else
573	# error "Port me"
574	# endif
575	}
576	else
577	{
578	/* reg1 = (uint32_t)(regflat >> 12) */
579	off = iemNativeEmitGpr32EqGprShiftRightImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT);
580	}
581	/* or reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
582	# if defined(RT_ARCH_AMD64)
583	pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
584	pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
585	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
586	# else
587	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
588	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
589	# endif
590
591	/*
592	* 3b. Calc pTlbe.
593	*/
594	uint32_t const offTlbEntries = offVCpuTlb + RT_UOFFSETOF(IEMTLB, aEntries);
595	# if defined(RT_ARCH_AMD64)
596	/* movzx reg2, byte reg1 */
597	off = iemNativeEmitLoadGprFromGpr8Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg1);
598	/* shl reg2, 5 ; reg2 = sizeof(IEMTLBENTRY) /
599	AssertCompileSize(IEMTLBENTRY, 32);
600	off = iemNativeEmitShiftGprLeftEx(pCodeBuf, off, pTlbState->idxReg2, 5);
601	/* lea reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
602	AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
603	pCodeBuf[off++] = pTlbState->idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_X \| X86_OP_REX_R;
604	pCodeBuf[off++] = 0x8d;
605	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, pTlbState->idxReg2 & 7, 4 /SIB/);
606	pCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, pTlbState->idxReg2 & 7, 0);
607	pCodeBuf[off++] = RT_BYTE1(offTlbEntries);
608	pCodeBuf[off++] = RT_BYTE2(offTlbEntries);
609	pCodeBuf[off++] = RT_BYTE3(offTlbEntries);
610	pCodeBuf[off++] = RT_BYTE4(offTlbEntries);
611
612	# elif defined(RT_ARCH_ARM64)
613	/* reg2 = (reg1 & 0xff) << 5 */
614	pCodeBuf[off++] = Armv8A64MkInstrUbfiz(pTlbState->idxReg2, pTlbState->idxReg1, 5, 8);
615	/* reg2 += offsetof(VMCPUCC, iem.s.DataTlb.aEntries) */
616	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, offTlbEntries, pTlbState->idxReg3 /iGprTmp/);
617	/* reg2 += pVCpu */
618	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, IEMNATIVE_REG_FIXED_PVMCPU);
619	# else
620	# error "Port me"
621	# endif
622
623	/*
624	* 3c. Compare the TLBE.uTag with the one from 2a (reg1).
625	*/
626	# if defined(RT_ARCH_AMD64)
627	/* cmp reg1, [reg2] */
628	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
629	pCodeBuf[off++] = 0x3b;
630	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
631	# elif defined(RT_ARCH_ARM64)
632	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
633	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
634	# else
635	# error "Port me"
636	# endif
637	/* jne tlbmiss */
638	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
639
640	/*
641	* 4. Check TLB page table level access flags and physical page revision #.
642	*/
643	/* mov reg1, mask */
644	AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
645	uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
646	uint64_t fTlbe = IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PT_NO_ACCESSED
647	\| fNoUser;
648	if (fAccess & IEM_ACCESS_TYPE_EXEC)
649	fTlbe \|= IEMTLBE_F_PT_NO_EXEC /\| IEMTLBE_F_PG_NO_READ?/;
650	if (fAccess & IEM_ACCESS_TYPE_READ)
651	fTlbe \|= IEMTLBE_F_PG_NO_READ;
652	if (fAccess & IEM_ACCESS_TYPE_WRITE)
653	fTlbe \|= IEMTLBE_F_PT_NO_WRITE \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY;
654	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, fTlbe);
655	# if defined(RT_ARCH_AMD64)
656	/* and reg1, [reg2->fFlagsAndPhysRev] */
657	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
658	pCodeBuf[off++] = 0x23;
659	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1,
660	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
661
662	/* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
663	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R);
664	pCodeBuf[off++] = 0x3b;
665	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
666	offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
667	# elif defined(RT_ARCH_ARM64)
668	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2,
669	RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
670	pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
671	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
672	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
673	# else
674	# error "Port me"
675	# endif
676	/* jne tlbmiss */
677	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
678
679	/*
680	* 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
681	* resulting pointer.
682	*
683	* For code TLB lookups we have some more work to do here to set various
684	* IEMCPU members and we return a GCPhys address rather than a host pointer.
685	*/
686	/* mov reg1, [reg2->pbMappingR3] */
687	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
688	RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
689	/* if (!reg1) goto tlbmiss; */
690	/** @todo eliminate the need for this test? */
691	off = iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, pTlbState->idxReg1,
692	true /f64Bit/, idxLabelTlbMiss);
693
694	if (a_fDataTlb)
695	{
696	if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
697	{
698	/* and result, 0xfff */
699	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
700	}
701	else
702	{
703	Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
704	/* result = regflat & 0xfff */
705	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
706	}
707
708	/* add result, reg1 */
709	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
710	}
711	else
712	{
713	/*
714	* Code TLB use a la iemOpcodeFetchBytesJmp - keep reg2 pointing to the TLBE.
715	*
716	* Note. We do not need to set offCurInstrStart or offInstrNextByte.
717	*/
718	# ifdef RT_ARCH_AMD64
719	uint8_t const idxReg3 = UINT8_MAX;
720	# else
721	uint8_t const idxReg3 = pTlbState->idxReg3;
722	# endif
723	/* Set pbInstrBuf first since we've got it loaded already. */
724	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
725	RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf), idxReg3);
726	/* Set uInstrBufPc to (FlatPC & ~GUEST_PAGE_OFFSET_MASK). */
727	off = iemNativeEmitGprEqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK);
728	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
729	RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc), idxReg3);
730	/* Set cbInstrBufTotal to GUEST_PAGE_SIZE. / /* @todo this is a simplifications. Calc right size using CS.LIM and EIP? */
731	off = iemNativeEmitStoreImmToVCpuU16Ex(pCodeBuf, off, GUEST_PAGE_SIZE, RT_UOFFSETOF(VMCPUCC, iem.s.cbInstrBufTotal),
732	pTlbState->idxReg1, idxReg3);
733	/* Now set GCPhysInstrBuf last as we'll be returning it in idxRegMemResult. */
734	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1,
735	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, GCPhys));
736	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
737	RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf), idxReg3);
738	if (!a_fNoReturn) /* (We skip this for iemNativeEmitBltLoadTlbAfterBranch.) */
739	{
740	/* Set idxRegMemResult. */
741	if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
742	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
743	else
744	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
745	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
746	}
747	}
748
749	# if 0
750	/*
751	* To verify the result we call a helper function.
752	*
753	* It's like the state logging, so parameters are passed on the stack.
754	* iemNativeHlpAsmSafeWrapCheckTlbLookup(pVCpu, result, addr, seg \| (cbMem << 8) \| (fAccess << 16))
755	*/
756	# ifdef RT_ARCH_AMD64
757	if (a_fDataTlb)
758	{
759	/* push seg \| (cbMem << 8) \| (fAccess << 16) */
760	pCodeBuf[off++] = 0x68;
761	pCodeBuf[off++] = iSegReg;
762	pCodeBuf[off++] = cbMem;
763	pCodeBuf[off++] = RT_BYTE1(fAccess);
764	pCodeBuf[off++] = RT_BYTE2(fAccess);
765	/* push pTlbState->idxRegPtr / immediate address. */
766	if (pTlbState->idxRegPtr != UINT8_MAX)
767	{
768	if (pTlbState->idxRegPtr >= 8)
769	pCodeBuf[off++] = X86_OP_REX_B;
770	pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
771	}
772	else
773	{
774	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
775	if (pTlbState->idxReg1 >= 8)
776	pCodeBuf[off++] = X86_OP_REX_B;
777	pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
778	}
779	/* push idxRegMemResult */
780	if (idxRegMemResult >= 8)
781	pCodeBuf[off++] = X86_OP_REX_B;
782	pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
783	/* push pVCpu */
784	pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
785	/* mov reg1, helper */
786	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup);
787	/* call [reg1] */
788	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_B);
789	pCodeBuf[off++] = 0xff;
790	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, pTlbState->idxReg1 & 7);
791	/* The stack is cleaned up by helper function. */
792	}
793
794	# else
795	# error "Port me"
796	# endif
797	# endif
798
799	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
800
801	return off;
802	}
803	#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
804
805
806	/** @} */
807
808	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h */
809

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerTlbLookup.h@ 103265

Download in other formats: