IEMN8veRecompilerTlbLookup.h@ 103799

Last change on this file since 103799 was 103622, checked in by vboxsync, 10 months ago
VMM/IEM: Obfuscate most variable indexes we pass around in strict builds so we easily catch register/variable index mixups. bugref:10371
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 38.3 KB

Line
1	/* $Id: IEMN8veRecompilerTlbLookup.h 103622 2024-03-01 00:42:36Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler TLB Lookup Code Emitter.
4	*/
5
6	/*
7	* Copyright (C) 2023-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35	#include "IEMN8veRecompilerEmit.h"
36
37
38	/** @defgroup grp_iem_n8ve_re_tlblookup Native Recompiler TLB Lookup Code Emitter
39	* @ingroup grp_iem_n8ve_re
40	* @{
41	*/
42
43	/*
44	* TLB Lookup config.
45	*/
46	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
47	# define IEMNATIVE_WITH_TLB_LOOKUP
48	#endif
49	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
50	# define IEMNATIVE_WITH_TLB_LOOKUP_FETCH
51	#endif
52	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
53	# define IEMNATIVE_WITH_TLB_LOOKUP_STORE
54	#endif
55	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
56	# define IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
57	#endif
58	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
59	# define IEMNATIVE_WITH_TLB_LOOKUP_PUSH
60	#endif
61	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
62	# define IEMNATIVE_WITH_TLB_LOOKUP_POP
63	#endif
64
65
66	/**
67	* This must be instantiate before branching off to the lookup code,
68	* so that register spilling and whatnot happens for everyone.
69	*/
70	typedef struct IEMNATIVEEMITTLBSTATE
71	{
72	bool const fSkip;
73	uint8_t const idxRegPtrHlp; /*< We don't support immediate variables with register assignment, so this a tmp reg alloc. /
74	uint8_t const idxRegPtr;
75	uint8_t const idxRegSegBase;
76	uint8_t const idxRegSegLimit;
77	uint8_t const idxRegSegAttrib;
78	uint8_t const idxReg1;
79	uint8_t const idxReg2;
80	#if defined(RT_ARCH_ARM64)
81	uint8_t const idxReg3;
82	#endif
83	uint64_t const uAbsPtr;
84
85	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint32_t *a_poff, uint8_t a_idxVarGCPtrMem,
86	uint8_t a_iSegReg, uint8_t a_cbMem, uint8_t a_offDisp = 0)
87	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
88	/* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
89	: fSkip( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
90	== kIemNativeVarKind_Immediate
91	&& ( (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
92	? (uint64_t)(UINT32_MAX - a_cbMem - a_offDisp)
93	: (uint64_t)(UINT64_MAX - a_cbMem - a_offDisp))
94	< a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue)
95	#else
96	: fSkip(true)
97	#endif
98	#if defined(RT_ARCH_AMD64) /* got good immediate encoding, otherwise we just load the address in a reg immediately. */
99	, idxRegPtrHlp(UINT8_MAX)
100	#else
101	, idxRegPtrHlp( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
102	!= kIemNativeVarKind_Immediate
103	\|\| fSkip
104	? UINT8_MAX
105	: iemNativeRegAllocTmpImm(a_pReNative, a_poff,
106	a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue))
107	#endif
108	, idxRegPtr( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
109	!= kIemNativeVarKind_Immediate
110	&& !fSkip
111	? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
112	true /fInitialized/, IEMNATIVE_CALL_ARG2_GREG)
113	: idxRegPtrHlp)
114	, idxRegSegBase(a_iSegReg == UINT8_MAX \|\| fSkip
115	? UINT8_MAX
116	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
117	, idxRegSegLimit((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
118	? UINT8_MAX
119	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
120	, idxRegSegAttrib((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
121	? UINT8_MAX
122	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
123	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
124	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
125	#if defined(RT_ARCH_ARM64)
126	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
127	#endif
128	, uAbsPtr( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
129	!= kIemNativeVarKind_Immediate
130	\|\| fSkip
131	? UINT64_MAX
132	: a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue)
133
134	{
135	RT_NOREF(a_cbMem, a_offDisp);
136	}
137
138	/* Alternative constructor for PUSH and POP where we don't have a GCPtrMem
139	variable, only a register derived from the guest RSP. */
140	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint8_t a_idxRegPtr, uint32_t *a_poff,
141	uint8_t a_iSegReg, uint8_t a_cbMem)
142	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
143	: fSkip(false)
144	#else
145	: fSkip(true)
146	#endif
147	, idxRegPtrHlp(UINT8_MAX)
148	, idxRegPtr(a_idxRegPtr)
149	, idxRegSegBase(a_iSegReg == UINT8_MAX \|\| fSkip
150	? UINT8_MAX
151	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
152	, idxRegSegLimit((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
153	? UINT8_MAX
154	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
155	, idxRegSegAttrib((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
156	? UINT8_MAX
157	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
158	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
159	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
160	#if defined(RT_ARCH_ARM64)
161	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
162	#endif
163	, uAbsPtr(UINT64_MAX)
164
165	{
166	RT_NOREF_PV(a_cbMem);
167	}
168
169	/* Alternative constructor for the code TLB lookups where we implictly use RIP
170	variable, only a register derived from the guest RSP. */
171	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, bool a_fFlat, uint32_t *a_poff)
172	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
173	: fSkip(false)
174	#else
175	: fSkip(true)
176	#endif
177	, idxRegPtrHlp(UINT8_MAX)
178	, idxRegPtr(iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, kIemNativeGstReg_Pc))
179	, idxRegSegBase(a_fFlat \|\| fSkip
180	? UINT8_MAX
181	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS)))
182	, idxRegSegLimit(/*a_fFlat \|\| fSkip
183	? UINT8_MAX
184	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS))*/
185	UINT8_MAX)
186	, idxRegSegAttrib(UINT8_MAX)
187	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
188	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
189	#if defined(RT_ARCH_ARM64)
190	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
191	#endif
192	, uAbsPtr(UINT64_MAX)
193
194	{
195	}
196
197	void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX, bool fIsCode = false) const
198	{
199	if (!fIsCode)
200	{
201	if (idxRegPtr != UINT8_MAX)
202	{
203	if (idxRegPtrHlp == UINT8_MAX)
204	{
205	if (idxVarGCPtrMem != UINT8_MAX)
206	iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
207	}
208	else
209	{
210	Assert(idxRegPtrHlp == idxRegPtr);
211	iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
212	}
213	}
214	else
215	Assert(idxRegPtrHlp == UINT8_MAX);
216	}
217	else
218	{
219	Assert(idxVarGCPtrMem == UINT8_MAX);
220	Assert(idxRegPtrHlp == UINT8_MAX);
221	iemNativeRegFreeTmp(a_pReNative, idxRegPtr); /* RIP */
222	}
223	if (idxRegSegBase != UINT8_MAX)
224	iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
225	if (idxRegSegLimit != UINT8_MAX)
226	iemNativeRegFreeTmp(a_pReNative, idxRegSegLimit);
227	if (idxRegSegAttrib != UINT8_MAX)
228	iemNativeRegFreeTmp(a_pReNative, idxRegSegAttrib);
229	#if defined(RT_ARCH_ARM64)
230	iemNativeRegFreeTmp(a_pReNative, idxReg3);
231	#endif
232	iemNativeRegFreeTmp(a_pReNative, idxReg2);
233	iemNativeRegFreeTmp(a_pReNative, idxReg1);
234
235	}
236
237	uint32_t getRegsNotToSave() const
238	{
239	if (!fSkip)
240	return RT_BIT_32(idxReg1)
241	\| RT_BIT_32(idxReg2)
242	#if defined(RT_ARCH_ARM64)
243	\| RT_BIT_32(idxReg3)
244	#endif
245	;
246	return 0;
247	}
248
249	/** This is only for avoid assertions. */
250	uint32_t getActiveRegsWithShadows(bool fCode = false) const
251	{
252	#ifdef VBOX_STRICT
253	if (!fSkip)
254	return (idxRegSegBase != UINT8_MAX ? RT_BIT_32(idxRegSegBase) : 0)
255	\| (idxRegSegLimit != UINT8_MAX ? RT_BIT_32(idxRegSegLimit) : 0)
256	\| (idxRegSegAttrib != UINT8_MAX ? RT_BIT_32(idxRegSegAttrib) : 0)
257	\| (fCode ? RT_BIT_32(idxRegPtr) : 0);
258	#else
259	RT_NOREF_PV(fCode);
260	#endif
261	return 0;
262	}
263	} IEMNATIVEEMITTLBSTATE;
264
265	DECLASM(void) iemNativeHlpAsmSafeWrapCheckTlbLookup(void);
266
267
268	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
269	template<bool const a_fDataTlb, bool const a_fNoReturn = false>
270	DECL_INLINE_THROW(uint32_t)
271	iemNativeEmitTlbLookup(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEMITTLBSTATE const * const pTlbState,
272	uint8_t iSegReg, uint8_t cbMem, uint8_t fAlignMask, uint32_t fAccess,
273	uint32_t idxLabelTlbLookup, uint32_t idxLabelTlbMiss, uint8_t idxRegMemResult,
274	uint8_t offDisp = 0)
275	{
276	Assert(!pTlbState->fSkip);
277	uint32_t const offVCpuTlb = a_fDataTlb ? RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb) : RT_UOFFSETOF(VMCPUCC, iem.s.CodeTlb);
278	# if defined(RT_ARCH_AMD64)
279	uint8_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 512);
280	# elif defined(RT_ARCH_ARM64)
281	uint32_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 64);
282	# endif
283
284	/*
285	* The expand down check isn't use all that much, so we emit here to keep
286	* the lookup straighter.
287	*/
288	/* check_expand_down: ; complicted! */
289	uint32_t const offCheckExpandDown = off;
290	uint32_t offFixupLimitDone = 0;
291	if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
292	{
293	off = iemNativeEmitBrkEx(pCodeBuf, off, 1); /** @todo this needs testing */
294	/* cmp seglim, regptr */
295	if (pTlbState->idxRegPtr != UINT8_MAX && offDisp == 0)
296	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxRegPtr);
297	else if (pTlbState->idxRegPtr == UINT8_MAX)
298	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
299	(uint32_t)(pTlbState->uAbsPtr + offDisp));
300	else if (cbMem == 1)
301	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxReg2);
302	else
303	{ /* use idxRegMemResult to calc the displaced address. */
304	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxRegPtr, offDisp);
305	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, idxRegMemResult);
306	}
307	/* ja tlbmiss */
308	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
309
310	/* reg1 = segattr & X86DESCATTR_D (0x4000) */
311	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib, X86DESCATTR_D);
312	/* xor reg1, X86DESCATTR_D */
313	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
314	/* shl reg1, 2 (16 - 14) */
315	AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
316	off = iemNativeEmitShiftGpr32LeftEx(pCodeBuf, off, pTlbState->idxReg1, 2);
317	/* dec reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
318	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, 1);
319	/* cmp reg1, reg2 (64-bit) / imm (32-bit) */
320	if (pTlbState->idxRegPtr != UINT8_MAX)
321	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1,
322	cbMem > 1 \|\| offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
323	else
324	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1,
325	(uint32_t)(pTlbState->uAbsPtr + offDisp + cbMem - 1)); /* fSkip=true on overflow. */
326	/* jbe tlbmiss */
327	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
328	/* jmp limitdone */
329	offFixupLimitDone = off;
330	off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, off /* ASSUME short jump suffices */);
331	}
332
333	/*
334	* tlblookup:
335	*/
336	iemNativeLabelDefine(pReNative, idxLabelTlbLookup, off);
337	# if defined(RT_ARCH_ARM64) && 0
338	off = iemNativeEmitBrkEx(pCodeBuf, off, 0);
339	# endif
340
341	/*
342	* 1. Segmentation.
343	*
344	* 1a. Check segment limit and attributes if non-flat 32-bit code. This is complicated.
345	*
346	* This can be skipped for code TLB lookups because limit is checked by jmp, call,
347	* ret, and iret prior to making it. It is also checked by the helpers prior to
348	* doing TLB loading.
349	*/
350	if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
351	{
352	/* Check that we've got a segment loaded and that it allows the access.
353	For write access this means a writable data segment.
354	For read-only accesses this means a readable code segment or any data segment. */
355	if (fAccess & IEM_ACCESS_TYPE_WRITE)
356	{
357	uint32_t const fMustBe1 = X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_WRITE;
358	uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE \| X86_SEL_TYPE_CODE;
359	/* reg1 = segattrs & (must1\|must0) */
360	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
361	pTlbState->idxRegSegAttrib, fMustBe1 \| fMustBe0);
362	/* cmp reg1, must1 */
363	AssertCompile(fMustBe1 <= UINT16_MAX);
364	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1);
365	/* jne tlbmiss */
366	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
367	}
368	else
369	{
370	/* U \| !P \|!DT \|!CD \| RW \|
371	16 \| 8 \| 4 \| 3 \| 1 \|
372	-------------------------------
373	0 \| 0 \| 0 \| 0 \| 0 \| execute-only code segment. - must be excluded
374	0 \| 0 \| 0 \| 0 \| 1 \| execute-read code segment.
375	0 \| 0 \| 0 \| 1 \| 0 \| read-only data segment.
376	0 \| 0 \| 0 \| 1 \| 1 \| read-write data segment. - last valid combination
377	*/
378	/* reg1 = segattrs & (relevant attributes) */
379	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib,
380	X86DESCATTR_UNUSABLE \| X86DESCATTR_P \| X86DESCATTR_DT
381	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE);
382	/* xor reg1, X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
383	; EO-code=0, ER-code=2, RO-data=8, RW-data=10 */
384	#ifdef RT_ARCH_ARM64
385	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_DT \| X86_SEL_TYPE_CODE);
386	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_P);
387	#else
388	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1,
389	X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_CODE);
390	#endif
391	/* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
392	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
393	/* cmp reg1, X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE */
394	AssertCompile(X86_SEL_TYPE_CODE == 8);
395	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_CODE);
396	/* ja tlbmiss */
397	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
398	}
399
400	/* If we're accessing more than one byte or if we're working with a non-zero offDisp,
401	put the last address we'll be accessing in idxReg2 (64-bit). */
402	if ((cbMem > 1 \|\| offDisp != 0) && pTlbState->idxRegPtr != UINT8_MAX)
403	{
404	if (!offDisp)
405	/* reg2 = regptr + cbMem - 1; 64-bit result so we can fend of wraparounds/overflows. */
406	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, pTlbState->idxReg2,/=/ pTlbState->idxRegPtr,/+/ cbMem - 1);
407	else
408	{
409	/* reg2 = (uint32_t)(regptr + offDisp) + cbMem - 1;. */
410	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off,
411	pTlbState->idxReg2,/=/ pTlbState->idxRegPtr,/+/ + offDisp);
412	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
413	}
414	}
415
416	/*
417	* Check the limit. If this is a write access, we know that it's a
418	* data segment and includes the expand_down bit. For read-only accesses
419	* we need to check that code/data=0 and expanddown=1 before continuing.
420	*/
421	if (fAccess & IEM_ACCESS_TYPE_WRITE)
422	{
423	/* test segattrs, X86_SEL_TYPE_DOWN */
424	AssertCompile(X86_SEL_TYPE_DOWN < 128);
425	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, pTlbState->idxRegSegAttrib, X86_SEL_TYPE_DOWN);
426	/* jnz check_expand_down */
427	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_ne);
428	}
429	else
430	{
431	/* reg1 = segattr & (code \| down) */
432	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
433	pTlbState->idxRegSegAttrib, X86_SEL_TYPE_CODE \| X86_SEL_TYPE_DOWN);
434	/* cmp reg1, down */
435	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_DOWN);
436	/* je check_expand_down */
437	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_e);
438	}
439
440	/* expand_up:
441	cmp seglim, regptr/reg2/imm */
442	if (pTlbState->idxRegPtr != UINT8_MAX)
443	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
444	cbMem > 1 \|\| offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
445	else
446	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
447	(uint32_t)pTlbState->uAbsPtr + offDisp + cbMem - 1U); /* fSkip=true on overflow. */
448	/* jbe tlbmiss */
449	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
450
451	/* limitdone: */
452	iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
453	}
454
455	/* 1b. Add the segment base. We use idxRegMemResult for the ptr register if
456	this step is required or if the address is a constant (simplicity) or
457	if offDisp is non-zero. */
458	uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX \|\| pTlbState->idxRegPtr == UINT8_MAX \|\| offDisp != 0
459	? idxRegMemResult : pTlbState->idxRegPtr;
460	if (iSegReg != UINT8_MAX)
461	{
462	Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
463	/* regflat = segbase + regptr/imm */
464	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
465	{
466	Assert(iSegReg >= X86_SREG_FS);
467	if (pTlbState->idxRegPtr != UINT8_MAX)
468	{
469	off = iemNativeEmitGprEqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
470	if (offDisp != 0)
471	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
472	}
473	else
474	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase,
475	pTlbState->uAbsPtr + offDisp);
476	}
477	else if (pTlbState->idxRegPtr != UINT8_MAX)
478	{
479	off = iemNativeEmitGpr32EqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
480	if (offDisp != 0)
481	off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
482	}
483	else
484	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr,
485	pTlbState->idxRegSegBase, (uint32_t)pTlbState->uAbsPtr + offDisp);
486	}
487	else if (pTlbState->idxRegPtr == UINT8_MAX)
488	{
489	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
490	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr + offDisp);
491	else
492	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, (uint32_t)pTlbState->uAbsPtr + offDisp);
493	}
494	else if (offDisp != 0)
495	{
496	Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
497	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
498	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
499	else
500	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
501	}
502	else
503	Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
504
505	/*
506	* 2. Check that the address doesn't cross a page boundrary and doesn't have alignment issues.
507	*
508	* 2a. Alignment check using fAlignMask.
509	*/
510	if (fAlignMask)
511	{
512	Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1));
513	Assert(fAlignMask < 128);
514	/* test regflat, fAlignMask */
515	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, idxRegFlatPtr, fAlignMask);
516	/* jnz tlbmiss */
517	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
518	}
519
520	/*
521	* 2b. Check that it's not crossing page a boundrary. This is implicit in
522	* the previous test if the alignment is same or larger than the type.
523	*/
524	if (cbMem > fAlignMask + 1)
525	{
526	/* reg1 = regflat & 0xfff */
527	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,/=/ idxRegFlatPtr,/&/ GUEST_PAGE_OFFSET_MASK);
528	/* cmp reg1, GUEST_PAGE_SIZE - cbMem */
529	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE);
530	/* ja tlbmiss */
531	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
532	}
533
534	/*
535	* 3. TLB lookup.
536	*
537	* 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG).
538	* In 64-bit mode we will also check for non-canonical addresses here.
539	*/
540	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
541	{
542	# if defined(RT_ARCH_AMD64)
543	/* mov reg1, regflat */
544	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
545	/* rol reg1, 16 */
546	off = iemNativeEmitRotateGprLeftEx(pCodeBuf, off, pTlbState->idxReg1, 16);
547	/** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
548	/* inc word reg1 */
549	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
550	if (pTlbState->idxReg1 >= 8)
551	pCodeBuf[off++] = X86_OP_REX_B;
552	pCodeBuf[off++] = 0xff;
553	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, pTlbState->idxReg1 & 7);
554	/* cmp word reg1, 1 */
555	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
556	if (pTlbState->idxReg1 >= 8)
557	pCodeBuf[off++] = X86_OP_REX_B;
558	pCodeBuf[off++] = 0x83;
559	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, pTlbState->idxReg1 & 7);
560	pCodeBuf[off++] = 1;
561	/* ja tlbmiss */
562	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
563	/* shr reg1, 16 + GUEST_PAGE_SHIFT */
564	off = iemNativeEmitShiftGprRightEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
565
566	# elif defined(RT_ARCH_ARM64)
567	/* lsr reg1, regflat, #48 */
568	pCodeBuf[off++] = Armv8A64MkInstrLslImm(pTlbState->idxReg1, idxRegFlatPtr, 4);
569	/* add reg1, reg1, #1 */
570	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(pTlbState->idxReg1, pTlbState->idxReg1, 1, false /f64Bit/);
571	/* tst reg1, #0xfffe */
572	Assert(Armv8A64ConvertImmRImmS2Mask32(14, 31) == 0xfffe);
573	pCodeBuf[off++] = Armv8A64MkInstrTstImm(pTlbState->idxReg1, 14, 31, false /f64Bit/);
574	/* b.nq tlbmiss */
575	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
576
577	/* ubfx reg1, regflat, #12, #36 */
578	pCodeBuf[off++] = Armv8A64MkInstrUbfx(pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT, 48 - GUEST_PAGE_SHIFT);
579	# else
580	# error "Port me"
581	# endif
582	}
583	else
584	{
585	/* reg1 = (uint32_t)(regflat >> 12) */
586	off = iemNativeEmitGpr32EqGprShiftRightImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT);
587	}
588	/* or reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
589	# if defined(RT_ARCH_AMD64)
590	pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
591	pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
592	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
593	# else
594	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
595	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
596	# endif
597
598	/*
599	* 3b. Calc pTlbe.
600	*/
601	uint32_t const offTlbEntries = offVCpuTlb + RT_UOFFSETOF(IEMTLB, aEntries);
602	# if defined(RT_ARCH_AMD64)
603	/* movzx reg2, byte reg1 */
604	off = iemNativeEmitLoadGprFromGpr8Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg1);
605	/* shl reg2, 5 ; reg2 = sizeof(IEMTLBENTRY) /
606	AssertCompileSize(IEMTLBENTRY, 32);
607	off = iemNativeEmitShiftGprLeftEx(pCodeBuf, off, pTlbState->idxReg2, 5);
608	/* lea reg2, [pVCpu->iem.s.DataTlb.aEntries + reg2] */
609	AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
610	pCodeBuf[off++] = pTlbState->idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_X \| X86_OP_REX_R;
611	pCodeBuf[off++] = 0x8d;
612	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, pTlbState->idxReg2 & 7, 4 /SIB/);
613	pCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, pTlbState->idxReg2 & 7, 0);
614	pCodeBuf[off++] = RT_BYTE1(offTlbEntries);
615	pCodeBuf[off++] = RT_BYTE2(offTlbEntries);
616	pCodeBuf[off++] = RT_BYTE3(offTlbEntries);
617	pCodeBuf[off++] = RT_BYTE4(offTlbEntries);
618
619	# elif defined(RT_ARCH_ARM64)
620	/* reg2 = (reg1 & 0xff) << 5 */
621	pCodeBuf[off++] = Armv8A64MkInstrUbfiz(pTlbState->idxReg2, pTlbState->idxReg1, 5, 8);
622	/* reg2 += offsetof(VMCPUCC, iem.s.DataTlb.aEntries) */
623	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, offTlbEntries, pTlbState->idxReg3 /iGprTmp/);
624	/* reg2 += pVCpu */
625	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, IEMNATIVE_REG_FIXED_PVMCPU);
626	# else
627	# error "Port me"
628	# endif
629
630	/*
631	* 3c. Compare the TLBE.uTag with the one from 2a (reg1).
632	*/
633	# if defined(RT_ARCH_AMD64)
634	/* cmp reg1, [reg2] */
635	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
636	pCodeBuf[off++] = 0x3b;
637	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
638	# elif defined(RT_ARCH_ARM64)
639	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
640	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
641	# else
642	# error "Port me"
643	# endif
644	/* jne tlbmiss */
645	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
646
647	/*
648	* 4. Check TLB page table level access flags and physical page revision #.
649	*/
650	/* mov reg1, mask */
651	AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
652	uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
653	uint64_t fTlbe = IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PT_NO_ACCESSED
654	\| fNoUser;
655	if (fAccess & IEM_ACCESS_TYPE_EXEC)
656	fTlbe \|= IEMTLBE_F_PT_NO_EXEC /\| IEMTLBE_F_PG_NO_READ?/;
657	if (fAccess & IEM_ACCESS_TYPE_READ)
658	fTlbe \|= IEMTLBE_F_PG_NO_READ;
659	if (fAccess & IEM_ACCESS_TYPE_WRITE)
660	fTlbe \|= IEMTLBE_F_PT_NO_WRITE \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY;
661	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, fTlbe);
662	# if defined(RT_ARCH_AMD64)
663	/* and reg1, [reg2->fFlagsAndPhysRev] */
664	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
665	pCodeBuf[off++] = 0x23;
666	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1,
667	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
668
669	/* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
670	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R);
671	pCodeBuf[off++] = 0x3b;
672	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
673	offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
674	# elif defined(RT_ARCH_ARM64)
675	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2,
676	RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
677	pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
678	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
679	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
680	# else
681	# error "Port me"
682	# endif
683	/* jne tlbmiss */
684	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
685
686	/*
687	* 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
688	* resulting pointer.
689	*
690	* For code TLB lookups we have some more work to do here to set various
691	* IEMCPU members and we return a GCPhys address rather than a host pointer.
692	*/
693	/* mov reg1, [reg2->pbMappingR3] */
694	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
695	RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
696	/* if (!reg1) goto tlbmiss; */
697	/** @todo eliminate the need for this test? */
698	off = iemNativeEmitTestIfGprIsZeroAndJmpToLabelEx(pReNative, pCodeBuf, off, pTlbState->idxReg1,
699	true /f64Bit/, idxLabelTlbMiss);
700
701	if (a_fDataTlb)
702	{
703	if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
704	{
705	/* and result, 0xfff */
706	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
707	}
708	else
709	{
710	Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
711	/* result = regflat & 0xfff */
712	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
713	}
714
715	/* add result, reg1 */
716	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
717	}
718	else
719	{
720	/*
721	* Code TLB use a la iemOpcodeFetchBytesJmp - keep reg2 pointing to the TLBE.
722	*
723	* Note. We do not need to set offCurInstrStart or offInstrNextByte.
724	*/
725	# ifdef RT_ARCH_AMD64
726	uint8_t const idxReg3 = UINT8_MAX;
727	# else
728	uint8_t const idxReg3 = pTlbState->idxReg3;
729	# endif
730	/* Set pbInstrBuf first since we've got it loaded already. */
731	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
732	RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf), idxReg3);
733	/* Set uInstrBufPc to (FlatPC & ~GUEST_PAGE_OFFSET_MASK). */
734	off = iemNativeEmitGprEqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK);
735	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
736	RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc), idxReg3);
737	/* Set cbInstrBufTotal to GUEST_PAGE_SIZE. / /* @todo this is a simplifications. Calc right size using CS.LIM and EIP? */
738	off = iemNativeEmitStoreImmToVCpuU16Ex(pCodeBuf, off, GUEST_PAGE_SIZE, RT_UOFFSETOF(VMCPUCC, iem.s.cbInstrBufTotal),
739	pTlbState->idxReg1, idxReg3);
740	/* Now set GCPhysInstrBuf last as we'll be returning it in idxRegMemResult. */
741	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1,
742	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, GCPhys));
743	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
744	RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf), idxReg3);
745	if (!a_fNoReturn) /* (We skip this for iemNativeEmitBltLoadTlbAfterBranch.) */
746	{
747	/* Set idxRegMemResult. */
748	if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
749	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
750	else
751	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
752	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
753	}
754	}
755
756	# if 0
757	/*
758	* To verify the result we call a helper function.
759	*
760	* It's like the state logging, so parameters are passed on the stack.
761	* iemNativeHlpAsmSafeWrapCheckTlbLookup(pVCpu, result, addr, seg \| (cbMem << 8) \| (fAccess << 16))
762	*/
763	# ifdef RT_ARCH_AMD64
764	if (a_fDataTlb)
765	{
766	/* push seg \| (cbMem << 8) \| (fAccess << 16) */
767	pCodeBuf[off++] = 0x68;
768	pCodeBuf[off++] = iSegReg;
769	pCodeBuf[off++] = cbMem;
770	pCodeBuf[off++] = RT_BYTE1(fAccess);
771	pCodeBuf[off++] = RT_BYTE2(fAccess);
772	/* push pTlbState->idxRegPtr / immediate address. */
773	if (pTlbState->idxRegPtr != UINT8_MAX)
774	{
775	if (pTlbState->idxRegPtr >= 8)
776	pCodeBuf[off++] = X86_OP_REX_B;
777	pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
778	}
779	else
780	{
781	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
782	if (pTlbState->idxReg1 >= 8)
783	pCodeBuf[off++] = X86_OP_REX_B;
784	pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
785	}
786	/* push idxRegMemResult */
787	if (idxRegMemResult >= 8)
788	pCodeBuf[off++] = X86_OP_REX_B;
789	pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
790	/* push pVCpu */
791	pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
792	/* mov reg1, helper */
793	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup);
794	/* call [reg1] */
795	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_B);
796	pCodeBuf[off++] = 0xff;
797	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, pTlbState->idxReg1 & 7);
798	/* The stack is cleaned up by helper function. */
799	}
800
801	# else
802	# error "Port me"
803	# endif
804	# endif
805
806	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
807
808	return off;
809	}
810	#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
811
812
813	/** @} */
814
815	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h */
816

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerTlbLookup.h@ 103799

Download in other formats: