VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllThreadedRecompiler.cpp@ 100694

Last change on this file since 100694 was 100694, checked in by vboxsync, 17 months ago

IEM/VMM: Deal with opcode checking cross page boundraries and tentativiely for branches. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 62.4 KB
Line 
1/* $Id: IEMAllThreadedRecompiler.cpp 100694 2023-07-25 10:34:22Z vboxsync $ */
2/** @file
3 * IEM - Instruction Decoding and Threaded Recompilation.
4 *
5 * Logging group IEM_RE_THREADED assignments:
6 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events. [same as IEM]
7 * - Flow (LogFlow) :
8 * - Level 2 (Log2) :
9 * - Level 3 (Log3) : More detailed enter/exit IEM state info. [same as IEM]
10 * - Level 4 (Log4) : Decoding mnemonics w/ EIP. [same as IEM]
11 * - Level 5 (Log5) : Decoding details. [same as IEM]
12 * - Level 6 (Log6) :
13 * - Level 7 (Log7) : TB obsoletion.
14 * - Level 8 (Log8) : TB compilation.
15 * - Level 9 (Log9) : TB exec.
16 * - Level 10 (Log10): TB block lookup.
17 * - Level 11 (Log11): TB block lookup details.
18 * - Level 12 (Log12): TB insertion.
19 */
20
21/*
22 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
23 *
24 * This file is part of VirtualBox base platform packages, as
25 * available from https://www.virtualbox.org.
26 *
27 * This program is free software; you can redistribute it and/or
28 * modify it under the terms of the GNU General Public License
29 * as published by the Free Software Foundation, in version 3 of the
30 * License.
31 *
32 * This program is distributed in the hope that it will be useful, but
33 * WITHOUT ANY WARRANTY; without even the implied warranty of
34 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
35 * General Public License for more details.
36 *
37 * You should have received a copy of the GNU General Public License
38 * along with this program; if not, see <https://www.gnu.org/licenses>.
39 *
40 * SPDX-License-Identifier: GPL-3.0-only
41 */
42
43
44/*********************************************************************************************************************************
45* Header Files *
46*********************************************************************************************************************************/
47#ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */
48# define LOG_GROUP LOG_GROUP_IEM_RE_THREADED
49#endif
50#define IEM_WITH_CODE_TLB_AND_OPCODE_BUF /* A bit hackish, but its all in IEMInline.h. */
51#define VMCPU_INCL_CPUM_GST_CTX
52#include <VBox/vmm/iem.h>
53#include <VBox/vmm/cpum.h>
54#include <VBox/vmm/apic.h>
55#include <VBox/vmm/pdm.h>
56#include <VBox/vmm/pgm.h>
57#include <VBox/vmm/iom.h>
58#include <VBox/vmm/em.h>
59#include <VBox/vmm/hm.h>
60#include <VBox/vmm/nem.h>
61#include <VBox/vmm/gim.h>
62#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
63# include <VBox/vmm/em.h>
64# include <VBox/vmm/hm_svm.h>
65#endif
66#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
67# include <VBox/vmm/hmvmxinline.h>
68#endif
69#include <VBox/vmm/tm.h>
70#include <VBox/vmm/dbgf.h>
71#include <VBox/vmm/dbgftrace.h>
72#ifndef TST_IEM_CHECK_MC
73# include "IEMInternal.h"
74#endif
75#include <VBox/vmm/vmcc.h>
76#include <VBox/log.h>
77#include <VBox/err.h>
78#include <VBox/param.h>
79#include <VBox/dis.h>
80#include <VBox/disopcode-x86-amd64.h>
81#include <iprt/asm-math.h>
82#include <iprt/assert.h>
83#include <iprt/mem.h>
84#include <iprt/string.h>
85#include <iprt/x86.h>
86
87#ifndef TST_IEM_CHECK_MC
88# include "IEMInline.h"
89# include "IEMOpHlp.h"
90# include "IEMMc.h"
91#endif
92
93#include "IEMThreadedFunctions.h"
94
95
96/*
97 * Narrow down configs here to avoid wasting time on unused configs here.
98 */
99
100#ifndef IEM_WITH_CODE_TLB
101# error The code TLB must be enabled for the recompiler.
102#endif
103
104#ifndef IEM_WITH_DATA_TLB
105# error The data TLB must be enabled for the recompiler.
106#endif
107
108#ifndef IEM_WITH_SETJMP
109# error The setjmp approach must be enabled for the recompiler.
110#endif
111
112
113/*********************************************************************************************************************************
114* Structures and Typedefs *
115*********************************************************************************************************************************/
116
117
118
119/*********************************************************************************************************************************
120* Internal Functions *
121*********************************************************************************************************************************/
122static bool iemThreadedCompileBeginEmitCallsComplications(PVMCPUCC pVCpu, PIEMTB pTb);
123static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb);
124
125
126/*********************************************************************************************************************************
127* Defined Constants And Macros *
128*********************************************************************************************************************************/
129#define g_apfnOneByteMap g_apfnIemThreadedRecompilerOneByteMap
130
131
132#undef IEM_MC_CALC_RM_EFF_ADDR
133#ifndef IEM_WITH_SETJMP
134# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
135 uint64_t uEffAddrInfo; \
136 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddrJmpEx(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff), &uEffAddrInfo))
137#else
138# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
139 uint64_t uEffAddrInfo; \
140 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmpEx(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &uEffAddrInfo))
141#endif
142
143#define IEM_MC2_BEGIN_EMIT_CALLS() \
144 { \
145 PIEMTB const pTb = pVCpu->iem.s.pCurTbR3; \
146 uint8_t const cbInstrMc2 = IEM_GET_INSTR_LEN(pVCpu); \
147 AssertMsg(pVCpu->iem.s.offOpcode == cbInstrMc2, \
148 ("%u vs %u (%04x:%08RX64)\n", pVCpu->iem.s.offOpcode, cbInstrMc2, \
149 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); \
150 \
151 /* No page crossing, right? */ \
152 uint16_t const offOpcodeMc2 = pTb->cbOpcodes; \
153 uint8_t const idxRangeMc2 = pTb->cRanges - 1; \
154 if ( !pVCpu->iem.s.fTbCrossedPage \
155 && !pVCpu->iem.s.fTbCheckOpcodes \
156 && !pVCpu->iem.s.fTbBranched \
157 && !(pTb->fFlags & IEMTB_F_CS_LIM_CHECKS)) \
158 { \
159 /** @todo Custom copy function, given range is 1 thru 15 bytes. */ \
160 memcpy(&pTb->pabOpcodes[offOpcodeMc2], pVCpu->iem.s.abOpcode, pVCpu->iem.s.offOpcode); \
161 pTb->cbOpcodes = offOpcodeMc2 + pVCpu->iem.s.offOpcode; \
162 pTb->aRanges[idxRangeMc2].cbOpcodes += cbInstrMc2; \
163 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated); \
164 } \
165 else if (iemThreadedCompileBeginEmitCallsComplications(pVCpu, pTb)) \
166 { /* likely */ } \
167 else \
168 return VINF_IEM_RECOMPILE_END_TB; \
169 \
170 do { } while (0)
171
172#define IEM_MC2_EMIT_CALL_0(a_enmFunction) do { \
173 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
174 \
175 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
176 pCall->enmFunction = a_enmFunction; \
177 pCall->offOpcode = offOpcodeMc2; \
178 pCall->cbOpcode = cbInstrMc2; \
179 pCall->idxRange = idxRangeMc2; \
180 pCall->auParams[0] = 0; \
181 pCall->auParams[1] = 0; \
182 pCall->auParams[2] = 0; \
183 } while (0)
184#define IEM_MC2_EMIT_CALL_1(a_enmFunction, a_uArg0) do { \
185 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
186 uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
187 \
188 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
189 pCall->enmFunction = a_enmFunction; \
190 pCall->offOpcode = offOpcodeMc2; \
191 pCall->cbOpcode = cbInstrMc2; \
192 pCall->idxRange = idxRangeMc2; \
193 pCall->auParams[0] = a_uArg0; \
194 pCall->auParams[1] = 0; \
195 pCall->auParams[2] = 0; \
196 } while (0)
197#define IEM_MC2_EMIT_CALL_2(a_enmFunction, a_uArg0, a_uArg1) do { \
198 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
199 uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
200 uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
201 \
202 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
203 pCall->enmFunction = a_enmFunction; \
204 pCall->offOpcode = offOpcodeMc2; \
205 pCall->cbOpcode = cbInstrMc2; \
206 pCall->idxRange = idxRangeMc2; \
207 pCall->auParams[0] = a_uArg0; \
208 pCall->auParams[1] = a_uArg1; \
209 pCall->auParams[2] = 0; \
210 } while (0)
211#define IEM_MC2_EMIT_CALL_3(a_enmFunction, a_uArg0, a_uArg1, a_uArg2) do { \
212 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
213 uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
214 uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
215 uint64_t const uArg2Check = (a_uArg2); RT_NOREF(uArg2Check); \
216 \
217 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
218 pCall->enmFunction = a_enmFunction; \
219 pCall->offOpcode = offOpcodeMc2; \
220 pCall->cbOpcode = cbInstrMc2; \
221 pCall->idxRange = idxRangeMc2; \
222 pCall->auParams[0] = a_uArg0; \
223 pCall->auParams[1] = a_uArg1; \
224 pCall->auParams[2] = a_uArg2; \
225 } while (0)
226
227#define IEM_MC2_END_EMIT_CALLS() \
228 pTb->cInstructions++; \
229 } while (0)
230
231
232/*
233 * IEM_MC_DEFER_TO_CIMPL_0 is easily wrapped up.
234 *
235 * Doing so will also take care of IEMOP_RAISE_DIVIDE_ERROR, IEMOP_RAISE_INVALID_LOCK_PREFIX,
236 * IEMOP_RAISE_INVALID_OPCODE and their users.
237 */
238#undef IEM_MC_DEFER_TO_CIMPL_0_RET
239#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_pfnCImpl) return iemThreadedRecompilerMcDeferToCImpl0(pVCpu, a_pfnCImpl)
240
241typedef IEM_CIMPL_DECL_TYPE_0(FNIEMCIMPL0);
242typedef FNIEMCIMPL0 *PFNIEMCIMPL0;
243
244DECLINLINE(VBOXSTRICTRC) iemThreadedRecompilerMcDeferToCImpl0(PVMCPUCC pVCpu, PFNIEMCIMPL0 pfnCImpl)
245{
246 return pfnCImpl(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
247}
248
249/**
250 * Calculates the effective address of a ModR/M memory operand, extended version
251 * for use in the recompilers.
252 *
253 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
254 *
255 * May longjmp on internal error.
256 *
257 * @return The effective address.
258 * @param pVCpu The cross context virtual CPU structure of the calling thread.
259 * @param bRm The ModRM byte.
260 * @param cbImmAndRspOffset - First byte: The size of any immediate
261 * following the effective address opcode bytes
262 * (only for RIP relative addressing).
263 * - Second byte: RSP displacement (for POP [ESP]).
264 * @param puInfo Extra info: 32-bit displacement (bits 31:0) and
265 * SIB byte (bits 39:32).
266 *
267 * @note This must be defined in a source file with matching
268 * IEM_WITH_CODE_TLB_AND_OPCODE_BUF define till the define is made default
269 * or implemented differently...
270 */
271RTGCPTR iemOpHlpCalcRmEffAddrJmpEx(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset, uint64_t *puInfo) IEM_NOEXCEPT_MAY_LONGJMP
272{
273 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
274# define SET_SS_DEF() \
275 do \
276 { \
277 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
278 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
279 } while (0)
280
281 if (!IEM_IS_64BIT_CODE(pVCpu))
282 {
283/** @todo Check the effective address size crap! */
284 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
285 {
286 uint16_t u16EffAddr;
287
288 /* Handle the disp16 form with no registers first. */
289 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
290 {
291 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
292 *puInfo = u16EffAddr;
293 }
294 else
295 {
296 /* Get the displacment. */
297 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
298 {
299 case 0: u16EffAddr = 0; break;
300 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
301 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
302 default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */
303 }
304 *puInfo = u16EffAddr;
305
306 /* Add the base and index registers to the disp. */
307 switch (bRm & X86_MODRM_RM_MASK)
308 {
309 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
310 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
311 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
312 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
313 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
314 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
315 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
316 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
317 }
318 }
319
320 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16 uInfo=%#RX64\n", u16EffAddr, *puInfo));
321 return u16EffAddr;
322 }
323
324 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
325 uint32_t u32EffAddr;
326 uint64_t uInfo;
327
328 /* Handle the disp32 form with no registers first. */
329 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
330 {
331 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
332 uInfo = u32EffAddr;
333 }
334 else
335 {
336 /* Get the register (or SIB) value. */
337 uInfo = 0;
338 switch ((bRm & X86_MODRM_RM_MASK))
339 {
340 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
341 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
342 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
343 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
344 case 4: /* SIB */
345 {
346 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
347 uInfo = (uint64_t)bSib << 32;
348
349 /* Get the index and scale it. */
350 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
351 {
352 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
353 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
354 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
355 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
356 case 4: u32EffAddr = 0; /*none */ break;
357 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
358 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
359 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
360 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
361 }
362 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
363
364 /* add base */
365 switch (bSib & X86_SIB_BASE_MASK)
366 {
367 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
368 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
369 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
370 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
371 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
372 case 5:
373 if ((bRm & X86_MODRM_MOD_MASK) != 0)
374 {
375 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
376 SET_SS_DEF();
377 }
378 else
379 {
380 uint32_t u32Disp;
381 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
382 u32EffAddr += u32Disp;
383 uInfo |= u32Disp;
384 }
385 break;
386 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
387 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
388 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
389 }
390 break;
391 }
392 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
393 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
394 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
395 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
396 }
397
398 /* Get and add the displacement. */
399 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
400 {
401 case 0:
402 break;
403 case 1:
404 {
405 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
406 u32EffAddr += i8Disp;
407 uInfo |= (uint32_t)(int32_t)i8Disp;
408 break;
409 }
410 case 2:
411 {
412 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
413 u32EffAddr += u32Disp;
414 uInfo |= u32Disp;
415 break;
416 }
417 default:
418 AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */
419 }
420 }
421
422 *puInfo = uInfo;
423 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32 uInfo=%#RX64\n", u32EffAddr, uInfo));
424 return u32EffAddr;
425 }
426
427 uint64_t u64EffAddr;
428 uint64_t uInfo;
429
430 /* Handle the rip+disp32 form with no registers first. */
431 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
432 {
433 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
434 uInfo = (uint32_t)u64EffAddr;
435 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
436 }
437 else
438 {
439 /* Get the register (or SIB) value. */
440 uInfo = 0;
441 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
442 {
443 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
444 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
445 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
446 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
447 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
448 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
449 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
450 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
451 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
452 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
453 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
454 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
455 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
456 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
457 /* SIB */
458 case 4:
459 case 12:
460 {
461 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
462 uInfo = (uint64_t)bSib << 32;
463
464 /* Get the index and scale it. */
465 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
466 {
467 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
468 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
469 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
470 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
471 case 4: u64EffAddr = 0; /*none */ break;
472 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
473 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
474 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
475 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
476 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
477 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
478 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
479 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
480 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
481 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
482 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
483 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
484 }
485 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
486
487 /* add base */
488 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
489 {
490 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
491 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
492 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
493 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
494 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
495 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
496 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
497 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
498 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
499 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
500 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
501 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
502 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
503 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
504 /* complicated encodings */
505 case 5:
506 case 13:
507 if ((bRm & X86_MODRM_MOD_MASK) != 0)
508 {
509 if (!pVCpu->iem.s.uRexB)
510 {
511 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
512 SET_SS_DEF();
513 }
514 else
515 u64EffAddr += pVCpu->cpum.GstCtx.r13;
516 }
517 else
518 {
519 uint32_t u32Disp;
520 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
521 u64EffAddr += (int32_t)u32Disp;
522 uInfo |= u32Disp;
523 }
524 break;
525 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
526 }
527 break;
528 }
529 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
530 }
531
532 /* Get and add the displacement. */
533 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
534 {
535 case 0:
536 break;
537 case 1:
538 {
539 int8_t i8Disp;
540 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
541 u64EffAddr += i8Disp;
542 uInfo |= (uint32_t)(int32_t)i8Disp;
543 break;
544 }
545 case 2:
546 {
547 uint32_t u32Disp;
548 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
549 u64EffAddr += (int32_t)u32Disp;
550 uInfo |= u32Disp;
551 break;
552 }
553 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
554 }
555
556 }
557
558 *puInfo = uInfo;
559 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
560 {
561 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr, uInfo));
562 return u64EffAddr;
563 }
564 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
565 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr & UINT32_MAX, uInfo));
566 return u64EffAddr & UINT32_MAX;
567}
568
569
570/*
571 * Include the "annotated" IEMAllInstructions*.cpp.h files.
572 */
573#include "IEMThreadedInstructions.cpp.h"
574
575/*
576 * Translation block management.
577 */
578typedef struct IEMTBCACHE
579{
580 uint32_t cHash;
581 uint32_t uHashMask;
582 PIEMTB apHash[_64K];
583} IEMTBCACHE;
584
585static IEMTBCACHE g_TbCache = { _64K, 0xffff, }; /**< Quick and dirty. */
586
587#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
588 ( ((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
589
590
591/**
592 * Allocate a translation block for threadeded recompilation.
593 *
594 * @returns Pointer to the translation block on success, NULL on failure.
595 * @param pVM The cross context virtual machine structure.
596 * @param pVCpu The cross context virtual CPU structure of the calling
597 * thread.
598 * @param GCPhysPc The physical address corresponding to RIP + CS.BASE.
599 * @param fExtraFlags Extra flags (IEMTB_F_XXX).
600 */
601static PIEMTB iemThreadedTbAlloc(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
602{
603 /*
604 * Just using the heap for now. Will make this more efficient and
605 * complicated later, don't worry. :-)
606 */
607 PIEMTB pTb = (PIEMTB)RTMemAlloc(sizeof(IEMTB));
608 if (pTb)
609 {
610 unsigned const cCalls = 128;
611 pTb->Thrd.paCalls = (PIEMTHRDEDCALLENTRY)RTMemAlloc(sizeof(IEMTHRDEDCALLENTRY) * cCalls);
612 if (pTb->Thrd.paCalls)
613 {
614 pTb->pabOpcodes = (uint8_t *)RTMemAlloc(cCalls * 16); /* This will be reallocated later. */
615 if (pTb->pabOpcodes)
616 {
617 pTb->Thrd.cAllocated = cCalls;
618 pTb->cbOpcodesAllocated = cCalls * 16;
619 pTb->Thrd.cCalls = 0;
620 pTb->cbOpcodes = 0;
621 pTb->pNext = NULL;
622 RTListInit(&pTb->LocalList);
623 pTb->GCPhysPc = GCPhysPc;
624 pTb->x86.fAttr = (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u;
625 pTb->fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags;
626
627 /* Init the first opcode range. */
628 pTb->cRanges = 1;
629 pTb->aRanges[0].cbOpcodes = 0;
630 pTb->aRanges[0].offOpcodes = 0;
631 pTb->aRanges[0].offPhysPage = GCPhysPc & GUEST_PAGE_OFFSET_MASK;
632 pTb->aRanges[0].u2Unused = 0;
633 pTb->aRanges[0].idxPhysPage = 0;
634 pTb->aGCPhysPages[0] = NIL_RTGCPHYS;
635 pTb->aGCPhysPages[1] = NIL_RTGCPHYS;
636
637 pVCpu->iem.s.cTbAllocs++;
638 return pTb;
639 }
640 RTMemFree(pTb->Thrd.paCalls);
641 }
642 RTMemFree(pTb);
643 }
644 RT_NOREF(pVM);
645 return NULL;
646}
647
648
649/**
650 * Frees pTb.
651 *
652 * @param pVM The cross context virtual machine structure.
653 * @param pVCpu The cross context virtual CPU structure of the calling
654 * thread.
655 * @param pTb The translation block to free..
656 */
657static void iemThreadedTbFree(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
658{
659 RT_NOREF(pVM);
660 AssertPtr(pTb);
661
662 AssertCompile(IEMTB_F_STATE_OBSOLETE == IEMTB_F_STATE_MASK);
663 pTb->fFlags |= IEMTB_F_STATE_OBSOLETE; /* works, both bits set */
664
665 /* Unlink it from the hash table: */
666 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
667 PIEMTB pTbCur = g_TbCache.apHash[idxHash];
668 if (pTbCur == pTb)
669 g_TbCache.apHash[idxHash] = pTb->pNext;
670 else
671 while (pTbCur)
672 {
673 PIEMTB const pNextTb = pTbCur->pNext;
674 if (pNextTb == pTb)
675 {
676 pTbCur->pNext = pTb->pNext;
677 break;
678 }
679 pTbCur = pNextTb;
680 }
681
682 /* Free it. */
683 RTMemFree(pTb->Thrd.paCalls);
684 pTb->Thrd.paCalls = NULL;
685
686 RTMemFree(pTb->pabOpcodes);
687 pTb->pabOpcodes = NULL;
688
689 RTMemFree(pTb);
690 pVCpu->iem.s.cTbFrees++;
691}
692
693
694/**
695 * Called by opcode verifier functions when they detect a problem.
696 */
697void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb)
698{
699 iemThreadedTbFree(pVCpu->CTX_SUFF(pVM), pVCpu, pTb);
700}
701
702
703static PIEMTB iemThreadedTbLookup(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
704{
705 uint32_t const fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags | IEMTB_F_STATE_READY;
706 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, fFlags, GCPhysPc);
707 Log10(("TB lookup: idxHash=%#x fFlags=%#x GCPhysPc=%RGp\n", idxHash, fFlags, GCPhysPc));
708 PIEMTB pTb = g_TbCache.apHash[idxHash];
709 while (pTb)
710 {
711 if (pTb->GCPhysPc == GCPhysPc)
712 {
713 if (pTb->fFlags == fFlags)
714 {
715 if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
716 {
717#ifdef VBOX_WITH_STATISTICS
718 pVCpu->iem.s.cTbLookupHits++;
719#endif
720 return pTb;
721 }
722 Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
723 }
724 else
725 Log11(("TB miss: fFlags: %#x, wanted %#x\n", pTb->fFlags, fFlags));
726 }
727 else
728 Log11(("TB miss: GCPhysPc: %#x, wanted %#x\n", pTb->GCPhysPc, GCPhysPc));
729
730 pTb = pTb->pNext;
731 }
732 RT_NOREF(pVM);
733 pVCpu->iem.s.cTbLookupMisses++;
734 return pTb;
735}
736
737
738static void iemThreadedTbAdd(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
739{
740 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
741 pTb->pNext = g_TbCache.apHash[idxHash];
742 g_TbCache.apHash[idxHash] = pTb;
743 Log12(("TB added: %p %RGp LB %#x fl=%#x idxHash=%#x\n", pTb, pTb->GCPhysPc, pTb->cbOpcodes, pTb->fFlags, idxHash));
744 RT_NOREF(pVM, pVCpu);
745}
746
747
748/*
749 * Real code.
750 */
751
752#ifdef LOG_ENABLED
753/**
754 * Logs the current instruction.
755 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
756 * @param pszFunction The IEM function doing the execution.
757 */
758static void iemThreadedLogCurInstr(PVMCPUCC pVCpu, const char *pszFunction) RT_NOEXCEPT
759{
760# ifdef IN_RING3
761 if (LogIs2Enabled())
762 {
763 char szInstr[256];
764 uint32_t cbInstr = 0;
765 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
766 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
767 szInstr, sizeof(szInstr), &cbInstr);
768
769 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
770 Log2(("**** %s fExec=%x pTb=%p\n"
771 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
772 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
773 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
774 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
775 " %s\n"
776 , pszFunction, pVCpu->iem.s.fExec, pVCpu->iem.s.pCurTbR3,
777 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
778 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
779 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
780 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
781 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
782 szInstr));
783
784 if (LogIs3Enabled())
785 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
786 }
787 else
788# endif
789 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
790 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
791}
792#endif /* LOG_ENABLED */
793
794
795static VBOXSTRICTRC iemThreadedCompileLongJumped(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
796{
797 RT_NOREF(pVM, pVCpu);
798 return rcStrict;
799}
800
801
802/**
803 * Initializes the decoder state when compiling TBs.
804 *
805 * This presumes that fExec has already be initialized.
806 *
807 * This is very similar to iemInitDecoder() and iemReInitDecoder(), so may need
808 * to apply fixes to them as well.
809 *
810 * @param pVCpu The cross context virtual CPU structure of the calling
811 * thread.
812 * @param fReInit Clear for the first call for a TB, set for subsequent calls
813 * from inside the compile loop where we can skip a couple of
814 * things.
815 */
816DECL_FORCE_INLINE(void) iemThreadedCompileInitDecoder(PVMCPUCC pVCpu, bool const fReInit)
817{
818 /* ASSUMES: That iemInitExec was already called and that anyone changing
819 CPU state affecting the fExec bits since then will have updated fExec! */
820 AssertMsg((pVCpu->iem.s.fExec & ~IEM_F_USER_OPTS) == iemCalcExecFlags(pVCpu),
821 ("fExec=%#x iemCalcExecModeFlags=%#x\n", pVCpu->iem.s.fExec, iemCalcExecFlags(pVCpu)));
822
823 IEMMODE const enmMode = IEM_GET_CPU_MODE(pVCpu);
824
825 /* Decoder state: */
826 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
827 pVCpu->iem.s.enmEffAddrMode = enmMode;
828 if (enmMode != IEMMODE_64BIT)
829 {
830 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
831 pVCpu->iem.s.enmEffOpSize = enmMode;
832 }
833 else
834 {
835 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
836 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
837 }
838 pVCpu->iem.s.fPrefixes = 0;
839 pVCpu->iem.s.uRexReg = 0;
840 pVCpu->iem.s.uRexB = 0;
841 pVCpu->iem.s.uRexIndex = 0;
842 pVCpu->iem.s.idxPrefix = 0;
843 pVCpu->iem.s.uVex3rdReg = 0;
844 pVCpu->iem.s.uVexLength = 0;
845 pVCpu->iem.s.fEvexStuff = 0;
846 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
847 pVCpu->iem.s.offModRm = 0;
848 pVCpu->iem.s.iNextMapping = 0;
849
850 if (!fReInit)
851 {
852 pVCpu->iem.s.cActiveMappings = 0;
853 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
854 pVCpu->iem.s.fEndTb = false;
855 pVCpu->iem.s.fTbCheckOpcodes = false;
856 pVCpu->iem.s.fTbBranched = false;
857 pVCpu->iem.s.fTbCrossedPage = false;
858 }
859 else
860 {
861 Assert(pVCpu->iem.s.cActiveMappings == 0);
862 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
863 Assert(pVCpu->iem.s.fEndTb == false);
864 Assert(pVCpu->iem.s.fTbCrossedPage == false);
865 }
866
867#ifdef DBGFTRACE_ENABLED
868 switch (IEM_GET_CPU_MODE(pVCpu))
869 {
870 case IEMMODE_64BIT:
871 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);
872 break;
873 case IEMMODE_32BIT:
874 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
875 break;
876 case IEMMODE_16BIT:
877 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
878 break;
879 }
880#endif
881}
882
883
884/**
885 * Initializes the opcode fetcher when starting the compilation.
886 *
887 * @param pVCpu The cross context virtual CPU structure of the calling
888 * thread.
889 */
890DECL_FORCE_INLINE(void) iemThreadedCompileInitOpcodeFetching(PVMCPUCC pVCpu)
891{
892 /* Almost everything is done by iemGetPcWithPhysAndCode() already. We just need to initialize the index into abOpcode. */
893#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
894 pVCpu->iem.s.offOpcode = 0;
895#else
896 RT_NOREF(pVCpu);
897#endif
898}
899
900
901/**
902 * Re-initializes the opcode fetcher between instructions while compiling.
903 *
904 * @param pVCpu The cross context virtual CPU structure of the calling
905 * thread.
906 */
907DECL_FORCE_INLINE(void) iemThreadedCompileReInitOpcodeFetching(PVMCPUCC pVCpu)
908{
909 if (pVCpu->iem.s.pbInstrBuf)
910 {
911 uint64_t off = pVCpu->cpum.GstCtx.rip;
912 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
913 off += pVCpu->cpum.GstCtx.cs.u64Base;
914 off -= pVCpu->iem.s.uInstrBufPc;
915 if (off < pVCpu->iem.s.cbInstrBufTotal)
916 {
917 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
918 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
919 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
920 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
921 else
922 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
923 }
924 else
925 {
926 pVCpu->iem.s.pbInstrBuf = NULL;
927 pVCpu->iem.s.offInstrNextByte = 0;
928 pVCpu->iem.s.offCurInstrStart = 0;
929 pVCpu->iem.s.cbInstrBuf = 0;
930 pVCpu->iem.s.cbInstrBufTotal = 0;
931 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
932 }
933 }
934 else
935 {
936 pVCpu->iem.s.offInstrNextByte = 0;
937 pVCpu->iem.s.offCurInstrStart = 0;
938 pVCpu->iem.s.cbInstrBuf = 0;
939 pVCpu->iem.s.cbInstrBufTotal = 0;
940#ifdef VBOX_STRICT
941 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
942#endif
943 }
944#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
945 pVCpu->iem.s.offOpcode = 0;
946#endif
947}
948
949
950DECLINLINE(void) iemThreadedCopyOpcodeBytesInline(PCVMCPUCC pVCpu, uint8_t *pbDst, uint8_t cbInstr)
951{
952 switch (cbInstr)
953 {
954 default: AssertMsgFailed(("%#x\n", cbInstr)); RT_FALL_THROUGH();
955 case 15: pbDst[14] = pVCpu->iem.s.abOpcode[14]; RT_FALL_THROUGH();
956 case 14: pbDst[13] = pVCpu->iem.s.abOpcode[13]; RT_FALL_THROUGH();
957 case 13: pbDst[12] = pVCpu->iem.s.abOpcode[12]; RT_FALL_THROUGH();
958 case 12: pbDst[11] = pVCpu->iem.s.abOpcode[11]; RT_FALL_THROUGH();
959 case 11: pbDst[10] = pVCpu->iem.s.abOpcode[10]; RT_FALL_THROUGH();
960 case 10: pbDst[9] = pVCpu->iem.s.abOpcode[9]; RT_FALL_THROUGH();
961 case 9: pbDst[8] = pVCpu->iem.s.abOpcode[8]; RT_FALL_THROUGH();
962 case 8: pbDst[7] = pVCpu->iem.s.abOpcode[7]; RT_FALL_THROUGH();
963 case 7: pbDst[6] = pVCpu->iem.s.abOpcode[6]; RT_FALL_THROUGH();
964 case 6: pbDst[5] = pVCpu->iem.s.abOpcode[5]; RT_FALL_THROUGH();
965 case 5: pbDst[4] = pVCpu->iem.s.abOpcode[4]; RT_FALL_THROUGH();
966 case 4: pbDst[3] = pVCpu->iem.s.abOpcode[3]; RT_FALL_THROUGH();
967 case 3: pbDst[2] = pVCpu->iem.s.abOpcode[2]; RT_FALL_THROUGH();
968 case 2: pbDst[1] = pVCpu->iem.s.abOpcode[1]; RT_FALL_THROUGH();
969 case 1: pbDst[0] = pVCpu->iem.s.abOpcode[0]; break;
970 }
971}
972
973
974/**
975 * Called by IEM_MC2_BEGIN_EMIT_CALLS() under one of these conditions:
976 *
977 * - CS LIM check required.
978 * - Must recheck opcode bytes.
979 * - Previous instruction branched.
980 * - TLB load detected, probably due to page crossing.
981 *
982 * @returns true if everything went well, false if we're out of space in the TB
983 * (e.g. opcode ranges).
984 * @param pVCpu The cross context virtual CPU structure of the calling
985 * thread.
986 * @param pTb The translation block being compiled.
987 */
988static bool iemThreadedCompileBeginEmitCallsComplications(PVMCPUCC pVCpu, PIEMTB pTb)
989{
990 Assert((pVCpu->iem.s.GCPhysInstrBuf & GUEST_PAGE_OFFSET_MASK) == 0);
991
992 /*
993 * Prepare call now, even before we know if can accept the instruction in this TB.
994 * This allows us amending parameters w/o making every case suffer.
995 */
996 uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu);
997 uint16_t const offOpcode = pTb->cbOpcodes;
998 uint8_t idxRange = pTb->cRanges - 1;
999
1000 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls];
1001 pCall->offOpcode = offOpcode;
1002 pCall->idxRange = idxRange;
1003 pCall->cbOpcode = cbInstr;
1004 pCall->auParams[0] = cbInstr;
1005 pCall->auParams[1] = idxRange;
1006 pCall->auParams[2] = offOpcode - pTb->aRanges[idxRange].offOpcodes;
1007
1008/** @todo check if we require IEMTB_F_CS_LIM_CHECKS for any new page we've
1009 * gotten onto. If we do, stop */
1010
1011 /*
1012 * Case 1: We've branched (RIP changed).
1013 *
1014 * Sub-case 1a: Same page, no TLB load, so fTbCrossedPage is false.
1015 * Req: 1 extra range, no extra phys.
1016 *
1017 * Sub-case 1b: Different page, so TLB load necessary and fTbCrossedPage is true.
1018 * Req: 1 extra range, probably 1 extra phys page entry.
1019 *
1020 * Sub-case 1c: Different page, so TLB load necessary and fTbCrossedPage is true,
1021 * but in addition we cross into the following page and require
1022 * another TLB load.
1023 * Req: 2 extra ranges, probably 2 extra phys page entries.
1024 *
1025 * Sub-case 1d: Same page, so no initial TLB load necessary, but we cross into
1026 * the following page and thus fTbCrossedPage is true.
1027 * Req: 2 extra ranges, probably 1 extra phys page entry.
1028 *
1029 * Note! We do not currently optimize branching to the next instruction (sorry
1030 * 32-bit PIC code). We could maybe do that in the branching code that sets (or not) fTbBranched.
1031 */
1032 if (pVCpu->iem.s.fTbBranched)
1033 {
1034AssertFailed(); /** @todo enable including branches in TBs and debug this code. */
1035 if ( !pVCpu->iem.s.fTbCrossedPage /* 1a */
1036 || pVCpu->iem.s.offCurInstrStart >= 0 /* 1b */ )
1037 {
1038 /* 1a + 1b - instruction fully within the branched to page. */
1039 Assert(pVCpu->iem.s.offCurInstrStart >= 0);
1040 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr <= GUEST_PAGE_SIZE);
1041
1042 /* Check that we've got a free range. */
1043 idxRange += 1;
1044 if (idxRange < RT_ELEMENTS(pTb->aRanges))
1045 { /* likely */ }
1046 else
1047 return false;
1048 pCall->idxRange = idxRange;
1049 pCall->auParams[1] = idxRange;
1050 pCall->auParams[2] = 0;
1051
1052 /* Check that we've got a free page slot. */
1053 AssertCompile(RT_ELEMENTS(pTb->aGCPhysPages) == 2);
1054 RTGCPHYS const GCPhysNew = pVCpu->iem.s.GCPhysInstrBuf & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1055 if ((pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysNew)
1056 pTb->aRanges[idxRange].idxPhysPage = 0;
1057 else if ( pTb->aGCPhysPages[0] == NIL_RTGCPHYS
1058 || pTb->aGCPhysPages[0] == GCPhysNew)
1059 {
1060 pTb->aGCPhysPages[0] = GCPhysNew;
1061 pTb->aRanges[idxRange].idxPhysPage = 1;
1062 }
1063 else if ( pTb->aGCPhysPages[1] == NIL_RTGCPHYS
1064 || pTb->aGCPhysPages[1] == GCPhysNew)
1065 {
1066 pTb->aGCPhysPages[1] = GCPhysNew;
1067 pTb->aRanges[idxRange].idxPhysPage = 2;
1068 }
1069 else
1070 return false;
1071
1072 /* Finish setting up the new range. */
1073 pTb->aRanges[idxRange].offPhysPage = pVCpu->iem.s.offCurInstrStart;
1074 pTb->aRanges[idxRange].offOpcodes = offOpcode;
1075 pTb->aRanges[idxRange].cbOpcodes = cbInstr;
1076 pTb->cRanges++;
1077
1078 /* Determin which function we need to load & check.
1079 Note! For jumps to a new page, we'll set both fTbBranched and
1080 fTbCrossedPage to avoid unnecessary TLB work for intra
1081 page branching */
1082 if (pVCpu->iem.s.fTbCrossedPage)
1083 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1084 ? kIemThreadedFunc_CheckCsLimAndOpcodesLoadingTlb
1085 : kIemThreadedFunc_CheckOpcodesLoadingTlb;
1086 else
1087 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1088 ? kIemThreadedFunc_CheckCsLimAndOpcodes
1089 : kIemThreadedFunc_CheckOpcodes;
1090 }
1091 else
1092 {
1093 /* 1c + 1d - instruction crosses pages. */
1094 Assert(pVCpu->iem.s.offCurInstrStart < 0);
1095 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr > 0);
1096
1097 /* Lazy bird: Check that this isn't case 1c, since we've already
1098 load the first physical address. End the TB and
1099 make it a case 2b instead.
1100
1101 Hmm. Too much bother to detect, so just do the same
1102 with case 1d as well. */
1103#if 0 /** @todo get back to this later when we've got the actual branch code in
1104 * place. */
1105 uint8_t const cbStartPage = (uint8_t)-pVCpu->iem.s.offCurInstrStart;
1106
1107 /* Check that we've got two free ranges. */
1108 if (idxRange + 2 < RT_ELEMENTS(pTb->aRanges))
1109 { /* likely */ }
1110 else
1111 return false;
1112 idxRange += 1;
1113 pCall->idxRange = idxRange;
1114 pCall->auParams[1] = idxRange;
1115 pCall->auParams[2] = 0;
1116
1117 /* ... */
1118
1119#else
1120 return false;
1121#endif
1122 }
1123 }
1124
1125 /*
1126 * Case 2: Page crossing.
1127 *
1128 * Sub-case 2a: The instruction starts on the first byte in the next page.
1129 *
1130 * Sub-case 2b: The instruction has opcode bytes in both the current and
1131 * following page.
1132 *
1133 * Both cases requires a new range table entry and probably a new physical
1134 * page entry. The difference is in which functions to emit and whether to
1135 * add bytes to the current range.
1136 */
1137 else if (pVCpu->iem.s.fTbCrossedPage)
1138 {
1139 /* Check that we've got a free range. */
1140 idxRange += 1;
1141 if (idxRange < RT_ELEMENTS(pTb->aRanges))
1142 { /* likely */ }
1143 else
1144 return false;
1145
1146 /* Check that we've got a free page slot. */
1147 AssertCompile(RT_ELEMENTS(pTb->aGCPhysPages) == 2);
1148 RTGCPHYS const GCPhysNew = pVCpu->iem.s.GCPhysInstrBuf & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1149 if ((pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysNew)
1150 pTb->aRanges[idxRange].idxPhysPage = 0;
1151 else if ( pTb->aGCPhysPages[0] == NIL_RTGCPHYS
1152 || pTb->aGCPhysPages[0] == GCPhysNew)
1153 {
1154 pTb->aGCPhysPages[0] = GCPhysNew;
1155 pTb->aRanges[idxRange].idxPhysPage = 1;
1156 }
1157 else if ( pTb->aGCPhysPages[1] == NIL_RTGCPHYS
1158 || pTb->aGCPhysPages[1] == GCPhysNew)
1159 {
1160 pTb->aGCPhysPages[1] = GCPhysNew;
1161 pTb->aRanges[idxRange].idxPhysPage = 2;
1162 }
1163 else
1164 return false;
1165
1166 if (((pTb->aRanges[idxRange - 1].offPhysPage + pTb->aRanges[idxRange - 1].cbOpcodes) & GUEST_PAGE_OFFSET_MASK) == 0)
1167 {
1168 Assert(pVCpu->iem.s.offCurInstrStart == 0);
1169 pCall->idxRange = idxRange;
1170 pCall->auParams[1] = idxRange;
1171 pCall->auParams[2] = 0;
1172
1173 /* Finish setting up the new range. */
1174 pTb->aRanges[idxRange].offPhysPage = pVCpu->iem.s.offCurInstrStart;
1175 pTb->aRanges[idxRange].offOpcodes = offOpcode;
1176 pTb->aRanges[idxRange].cbOpcodes = cbInstr;
1177 pTb->cRanges++;
1178
1179 /* Determin which function we need to load & check. */
1180 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1181 ? kIemThreadedFunc_CheckCsLimAndOpcodesLoadingTlb
1182 : kIemThreadedFunc_CheckOpcodesLoadingTlb;
1183 }
1184 else
1185 {
1186 Assert(pVCpu->iem.s.offCurInstrStart < 0);
1187 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr > 0);
1188 uint8_t const cbStartPage = (uint8_t)-pVCpu->iem.s.offCurInstrStart;
1189 pCall->auParams[0] |= (uint64_t)cbStartPage << 32;
1190
1191 /* We've good. Split the instruction over the old and new range table entries. */
1192 pTb->aRanges[idxRange - 1].cbOpcodes += cbStartPage;
1193
1194 pTb->aRanges[idxRange].offPhysPage = 0;
1195 pTb->aRanges[idxRange].offOpcodes = offOpcode + cbStartPage;
1196 pTb->aRanges[idxRange].cbOpcodes = cbInstr - cbStartPage;
1197 pTb->cRanges++;
1198
1199 /* Determin which function we need to load & check. */
1200 if (pVCpu->iem.s.fTbCheckOpcodes)
1201 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1202 ? kIemThreadedFunc_CheckCsLimAndOpcodesAcrossPageLoadingTlb
1203 : kIemThreadedFunc_CheckOpcodesAcrossPageLoadingTlb;
1204 else
1205 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1206 ? kIemThreadedFunc_CheckCsLimAndOpcodesOnNextPageLoadingTlb
1207 : kIemThreadedFunc_CheckOpcodesOnNextPageLoadingTlb;
1208 }
1209 }
1210
1211 /*
1212 * Regular case: No new range required.
1213 */
1214 else
1215 {
1216 Assert(pVCpu->iem.s.fTbCheckOpcodes || (pTb->fFlags & IEMTB_F_CS_LIM_CHECKS));
1217 if (pVCpu->iem.s.fTbCheckOpcodes)
1218 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1219 ? kIemThreadedFunc_CheckCsLimAndOpcodes
1220 : kIemThreadedFunc_CheckOpcodes;
1221 else
1222 pCall->enmFunction = kIemThreadedFunc_CheckCsLim;
1223
1224 iemThreadedCopyOpcodeBytesInline(pVCpu, &pTb->pabOpcodes[offOpcode], cbInstr);
1225 pTb->cbOpcodes = offOpcode + cbInstr;
1226 pTb->aRanges[idxRange].cbOpcodes += cbInstr;
1227 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1228 }
1229
1230 /*
1231 * Commit the call.
1232 */
1233 pTb->Thrd.cCalls++;
1234
1235 /*
1236 * Clear state.
1237 */
1238 pVCpu->iem.s.fTbBranched = false;
1239 pVCpu->iem.s.fTbCrossedPage = false;
1240 pVCpu->iem.s.fTbCheckOpcodes = false;
1241
1242 /*
1243 * Copy opcode bytes.
1244 */
1245 iemThreadedCopyOpcodeBytesInline(pVCpu, &pTb->pabOpcodes[offOpcode], cbInstr);
1246 pTb->cbOpcodes = offOpcode + cbInstr;
1247 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1248
1249 return true;
1250}
1251
1252
1253
1254/**
1255 * Compiles a new TB and executes it.
1256 *
1257 * We combine compilation and execution here as it makes it simpler code flow
1258 * in the main loop and it allows interpreting while compiling if we want to
1259 * explore that option.
1260 *
1261 * @returns Strict VBox status code.
1262 * @param pVM The cross context virtual machine structure.
1263 * @param pVCpu The cross context virtual CPU structure of the calling
1264 * thread.
1265 * @param GCPhysPc The physical address corresponding to the current
1266 * RIP+CS.BASE.
1267 * @param fExtraFlags Extra translation block flags: IEMTB_F_TYPE_THREADED and
1268 * maybe IEMTB_F_RIP_CHECKS.
1269 */
1270static VBOXSTRICTRC iemThreadedCompile(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
1271{
1272 /*
1273 * Allocate a new translation block.
1274 */
1275 PIEMTB pTb = iemThreadedTbAlloc(pVM, pVCpu, GCPhysPc, fExtraFlags | IEMTB_F_STATE_COMPILING);
1276 AssertReturn(pTb, VERR_IEM_TB_ALLOC_FAILED);
1277
1278 /* Set the current TB so iemThreadedCompileLongJumped and the CIMPL
1279 functions may get at it. */
1280 pVCpu->iem.s.pCurTbR3 = pTb;
1281
1282 /*
1283 * Now for the recomplication. (This mimicks IEMExecLots in many ways.)
1284 */
1285 iemThreadedCompileInitDecoder(pVCpu, false /*fReInit*/);
1286 iemThreadedCompileInitOpcodeFetching(pVCpu);
1287 VBOXSTRICTRC rcStrict;
1288 for (;;)
1289 {
1290 /* Process the next instruction. */
1291#ifdef LOG_ENABLED
1292 iemThreadedLogCurInstr(pVCpu, "CC");
1293 uint16_t const uCsLog = pVCpu->cpum.GstCtx.cs.Sel;
1294 uint64_t const uRipLog = pVCpu->cpum.GstCtx.rip;
1295#endif
1296 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
1297 uint16_t const cCallsPrev = pTb->Thrd.cCalls;
1298
1299 rcStrict = FNIEMOP_CALL(g_apfnIemThreadedRecompilerOneByteMap[b]);
1300 if ( rcStrict == VINF_SUCCESS
1301 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS
1302 && !pVCpu->iem.s.fEndTb)
1303 {
1304 Assert(pTb->Thrd.cCalls > cCallsPrev);
1305 Assert(cCallsPrev - pTb->Thrd.cCalls < 5);
1306
1307 pVCpu->iem.s.cInstructions++;
1308 }
1309 else
1310 {
1311 Log8(("%04x:%08RX64: End TB - %u instr, %u calls, rc=%d\n",
1312 uCsLog, uRipLog, pTb->cInstructions, pTb->Thrd.cCalls, VBOXSTRICTRC_VAL(rcStrict)));
1313 if (rcStrict == VINF_IEM_RECOMPILE_END_TB)
1314 rcStrict = VINF_SUCCESS;
1315
1316 if (pTb->Thrd.cCalls > 0)
1317 {
1318 if (cCallsPrev != pTb->Thrd.cCalls)
1319 pVCpu->iem.s.cInstructions++;
1320 break;
1321 }
1322
1323 pVCpu->iem.s.pCurTbR3 = NULL;
1324 iemThreadedTbFree(pVM, pVCpu, pTb);
1325 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1326 }
1327
1328 /* Still space in the TB? */
1329 if ( pTb->Thrd.cCalls + 5 < pTb->Thrd.cAllocated
1330 && pTb->cbOpcodes + 16 <= pTb->cbOpcodesAllocated)
1331 iemThreadedCompileInitDecoder(pVCpu, true /*fReInit*/);
1332 else
1333 {
1334 Log8(("%04x:%08RX64: End TB - %u instr, %u calls, %u opcode bytes - full\n",
1335 uCsLog, uRipLog, pTb->cInstructions, pTb->Thrd.cCalls, pTb->cbOpcodes));
1336 break;
1337 }
1338 iemThreadedCompileReInitOpcodeFetching(pVCpu);
1339 }
1340
1341 /*
1342 * Complete the TB and link it.
1343 */
1344 pTb->fFlags = (pTb->fFlags & ~IEMTB_F_STATE_MASK) | IEMTB_F_STATE_READY;
1345 iemThreadedTbAdd(pVM, pVCpu, pTb);
1346
1347#ifdef IEM_COMPILE_ONLY_MODE
1348 /*
1349 * Execute the translation block.
1350 */
1351#endif
1352
1353 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1354}
1355
1356
1357/**
1358 * Executes a translation block.
1359 *
1360 * @returns Strict VBox status code.
1361 * @param pVCpu The cross context virtual CPU structure of the calling
1362 * thread.
1363 * @param pTb The translation block to execute.
1364 */
1365static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb)
1366{
1367 /* Check the opcodes in the first page before starting execution. */
1368 uint32_t const cbLeadOpcodes = RT_MIN(pTb->cbOpcodes, pVCpu->iem.s.cbInstrBufTotal - pVCpu->iem.s.offInstrNextByte);
1369 if (memcmp(pTb->pabOpcodes, &pVCpu->iem.s.pbInstrBuf[pVCpu->iem.s.offInstrNextByte], cbLeadOpcodes) == 0)
1370 Assert( pTb->cbOpcodes == cbLeadOpcodes
1371 || cbLeadOpcodes == (GUEST_PAGE_SIZE - (pTb->GCPhysPc & GUEST_PAGE_OFFSET_MASK)));
1372 else
1373 {
1374 Log7(("TB obsolete: %p GCPhys=%RGp\n", pTb, pTb->GCPhysPc));
1375 iemThreadedTbFree(pVCpu->pVMR3, pVCpu, pTb);
1376 return VINF_SUCCESS;
1377 }
1378
1379 /* Set the current TB so CIMPL function may get at it. */
1380 pVCpu->iem.s.pCurTbR3 = pTb;
1381 pVCpu->iem.s.cTbExec++;
1382
1383 /* The execution loop. */
1384 PCIEMTHRDEDCALLENTRY pCallEntry = pTb->Thrd.paCalls;
1385 uint32_t cCallsLeft = pTb->Thrd.cCalls;
1386 while (cCallsLeft-- > 0)
1387 {
1388#ifdef LOG_ENABLED
1389 iemThreadedLogCurInstr(pVCpu, "EX");
1390 Log9(("%04x:%08RX64: #%d - %d %s\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
1391 pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->enmFunction, g_apszIemThreadedFunctions[pCallEntry->enmFunction]));
1392#endif
1393 VBOXSTRICTRC const rcStrict = g_apfnIemThreadedFunctions[pCallEntry->enmFunction](pVCpu,
1394 pCallEntry->auParams[0],
1395 pCallEntry->auParams[1],
1396 pCallEntry->auParams[2]);
1397
1398 if (RT_LIKELY( rcStrict == VINF_SUCCESS
1399 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
1400 pCallEntry++;
1401 else
1402 {
1403 pVCpu->iem.s.pCurTbR3 = NULL;
1404
1405 /* Some status codes are just to get us out of this loop and
1406 continue in a different translation block. */
1407 if (rcStrict == VINF_IEM_REEXEC_MODE_CHANGED)
1408 return iemExecStatusCodeFiddling(pVCpu, VINF_SUCCESS);
1409 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1410 }
1411 }
1412
1413 pVCpu->iem.s.pCurTbR3 = NULL;
1414 return VINF_SUCCESS;
1415}
1416
1417
1418/**
1419 * This is called when the PC doesn't match the current pbInstrBuf.
1420 *
1421 * Upon return, we're ready for opcode fetching. But please note that
1422 * pbInstrBuf can be NULL iff the memory doesn't have readable backing (i.e.
1423 * MMIO or unassigned).
1424 */
1425static RTGCPHYS iemGetPcWithPhysAndCodeMissed(PVMCPUCC pVCpu)
1426{
1427 pVCpu->iem.s.pbInstrBuf = NULL;
1428 pVCpu->iem.s.offCurInstrStart = 0;
1429 pVCpu->iem.s.offInstrNextByte = 0;
1430 iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
1431 return pVCpu->iem.s.GCPhysInstrBuf + pVCpu->iem.s.offCurInstrStart;
1432}
1433
1434
1435/** @todo need private inline decl for throw/nothrow matching IEM_WITH_SETJMP? */
1436DECL_FORCE_INLINE_THROW(RTGCPHYS) iemGetPcWithPhysAndCode(PVMCPUCC pVCpu)
1437{
1438 /*
1439 * Set uCurTbStartPc to RIP and calc the effective PC.
1440 */
1441 uint64_t uPc = pVCpu->cpum.GstCtx.rip;
1442 pVCpu->iem.s.uCurTbStartPc = uPc;
1443 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
1444 uPc += pVCpu->cpum.GstCtx.cs.u64Base;
1445
1446 /*
1447 * Advance within the current buffer (PAGE) when possible.
1448 */
1449 if (pVCpu->iem.s.pbInstrBuf)
1450 {
1451 uint64_t off = uPc - pVCpu->iem.s.uInstrBufPc;
1452 if (off < pVCpu->iem.s.cbInstrBufTotal)
1453 {
1454 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1455 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1456 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1457 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1458 else
1459 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1460
1461 return pVCpu->iem.s.GCPhysInstrBuf + off;
1462 }
1463 }
1464 return iemGetPcWithPhysAndCodeMissed(pVCpu);
1465}
1466
1467
1468/**
1469 * Determines the extra IEMTB_F_XXX flags.
1470 *
1471 * @returns IEMTB_F_TYPE_THREADED and maybe IEMTB_F_RIP_CHECKS.
1472 * @param pVCpu The cross context virtual CPU structure of the calling
1473 * thread.
1474 */
1475DECL_FORCE_INLINE(uint32_t) iemGetTbFlagsForCurrentPc(PVMCPUCC pVCpu)
1476{
1477 /*
1478 * Return IEMTB_F_RIP_CHECKS if the current PC is invalid or if it is
1479 * likely to go invalid before the end of the translation block.
1480 */
1481 if (IEM_IS_64BIT_CODE(pVCpu))
1482 return IEMTB_F_TYPE_THREADED;
1483
1484 if (RT_LIKELY( pVCpu->cpum.GstCtx.eip < pVCpu->cpum.GstCtx.cs.u32Limit
1485 && pVCpu->cpum.GstCtx.eip - pVCpu->cpum.GstCtx.cs.u32Limit >= X86_PAGE_SIZE))
1486 return IEMTB_F_TYPE_THREADED;
1487
1488 return IEMTB_F_TYPE_THREADED | IEMTB_F_CS_LIM_CHECKS;
1489}
1490
1491
1492VMMDECL(VBOXSTRICTRC) IEMExecRecompilerThreaded(PVMCC pVM, PVMCPUCC pVCpu)
1493{
1494 /*
1495 * See if there is an interrupt pending in TRPM, inject it if we can.
1496 */
1497 if (!TRPMHasTrap(pVCpu))
1498 { /* likely */ }
1499 else
1500 {
1501 VBOXSTRICTRC rcStrict = iemExecInjectPendingTrap(pVCpu);
1502 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1503 { /*likely */ }
1504 else
1505 return rcStrict;
1506 }
1507
1508 /*
1509 * Init the execution environment.
1510 */
1511 iemInitExec(pVCpu, 0 /*fExecOpts*/);
1512
1513 /*
1514 * Run-loop.
1515 *
1516 * If we're using setjmp/longjmp we combine all the catching here to avoid
1517 * having to call setjmp for each block we're executing.
1518 */
1519 for (;;)
1520 {
1521 PIEMTB pTb = NULL;
1522 VBOXSTRICTRC rcStrict;
1523 IEM_TRY_SETJMP(pVCpu, rcStrict)
1524 {
1525 uint32_t const cPollRate = 511; /* EM.cpp passes 4095 to IEMExecLots, so an eigth of that seems reasonable for now. */
1526 for (uint32_t iIterations = 0; ; iIterations++)
1527 {
1528 /* Translate PC to physical address, we'll need this for both lookup and compilation. */
1529 RTGCPHYS const GCPhysPc = iemGetPcWithPhysAndCode(pVCpu);
1530 uint32_t const fExtraFlags = iemGetTbFlagsForCurrentPc(pVCpu);
1531
1532 pTb = iemThreadedTbLookup(pVM, pVCpu, GCPhysPc, fExtraFlags);
1533 if (pTb)
1534 rcStrict = iemThreadedTbExec(pVCpu, pTb);
1535 else
1536 rcStrict = iemThreadedCompile(pVM, pVCpu, GCPhysPc, fExtraFlags);
1537 if (rcStrict == VINF_SUCCESS)
1538 {
1539 Assert(pVCpu->iem.s.cActiveMappings == 0);
1540
1541 uint64_t fCpu = pVCpu->fLocalForcedActions;
1542 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
1543 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
1544 | VMCPU_FF_TLB_FLUSH
1545 | VMCPU_FF_UNHALT );
1546 if (RT_LIKELY( ( !fCpu
1547 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
1548 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
1549 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
1550 {
1551 if (RT_LIKELY( (iIterations & cPollRate) != 0
1552 || !TMTimerPollBool(pVM, pVCpu)))
1553 {
1554
1555 }
1556 else
1557 return VINF_SUCCESS;
1558 }
1559 else
1560 return VINF_SUCCESS;
1561 }
1562 else
1563 return rcStrict;
1564 }
1565 }
1566 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
1567 {
1568 pVCpu->iem.s.cLongJumps++;
1569 if (pVCpu->iem.s.cActiveMappings > 0)
1570 iemMemRollback(pVCpu);
1571
1572 /* If pTb isn't NULL we're in iemThreadedTbExec. */
1573 if (!pTb)
1574 {
1575 /* If pCurTbR3 is NULL, we're in iemGetPcWithPhysAndCode.*/
1576 pTb = pVCpu->iem.s.pCurTbR3;
1577 if (pTb)
1578 {
1579 /* If the pCurTbR3 block is in compiling state, we're in iemThreadedCompile,
1580 otherwise it's iemThreadedTbExec inside iemThreadedCompile (compile option). */
1581 if ((pTb->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_COMPILING)
1582 return iemThreadedCompileLongJumped(pVM, pVCpu, rcStrict);
1583 }
1584 }
1585 return rcStrict;
1586 }
1587 IEM_CATCH_LONGJMP_END(pVCpu);
1588 }
1589}
1590
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette