VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllThreadedRecompiler.cpp@ 100671

Last change on this file since 100671 was 100671, checked in by vboxsync, 17 months ago

VMM/IEM: Fix TB revalidation prior to execution. The code is still incomplete, though. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 50.5 KB
Line 
1/* $Id: IEMAllThreadedRecompiler.cpp 100671 2023-07-21 00:28:24Z vboxsync $ */
2/** @file
3 * IEM - Instruction Decoding and Threaded Recompilation.
4 *
5 * Logging group IEM_RE_THREADED assignments:
6 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events. [same as IEM]
7 * - Flow (LogFlow) :
8 * - Level 2 (Log2) :
9 * - Level 3 (Log3) : More detailed enter/exit IEM state info. [same as IEM]
10 * - Level 4 (Log4) : Decoding mnemonics w/ EIP. [same as IEM]
11 * - Level 5 (Log5) : Decoding details. [same as IEM]
12 * - Level 6 (Log6) :
13 * - Level 7 (Log7) :
14 * - Level 8 (Log8) : TB compilation.
15 * - Level 9 (Log9) : TB exec.
16 * - Level 10 (Log10): TB block lookup.
17 * - Level 11 (Log11): TB block lookup.
18 * - Level 12 (Log12): TB insertion.
19 */
20
21/*
22 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
23 *
24 * This file is part of VirtualBox base platform packages, as
25 * available from https://www.virtualbox.org.
26 *
27 * This program is free software; you can redistribute it and/or
28 * modify it under the terms of the GNU General Public License
29 * as published by the Free Software Foundation, in version 3 of the
30 * License.
31 *
32 * This program is distributed in the hope that it will be useful, but
33 * WITHOUT ANY WARRANTY; without even the implied warranty of
34 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
35 * General Public License for more details.
36 *
37 * You should have received a copy of the GNU General Public License
38 * along with this program; if not, see <https://www.gnu.org/licenses>.
39 *
40 * SPDX-License-Identifier: GPL-3.0-only
41 */
42
43
44/*********************************************************************************************************************************
45* Header Files *
46*********************************************************************************************************************************/
47#ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */
48# define LOG_GROUP LOG_GROUP_IEM_RE_THREADED
49#endif
50#define IEM_WITH_CODE_TLB_AND_OPCODE_BUF /* A bit hackish, but its all in IEMInline.h. */
51#define VMCPU_INCL_CPUM_GST_CTX
52#include <VBox/vmm/iem.h>
53#include <VBox/vmm/cpum.h>
54#include <VBox/vmm/apic.h>
55#include <VBox/vmm/pdm.h>
56#include <VBox/vmm/pgm.h>
57#include <VBox/vmm/iom.h>
58#include <VBox/vmm/em.h>
59#include <VBox/vmm/hm.h>
60#include <VBox/vmm/nem.h>
61#include <VBox/vmm/gim.h>
62#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
63# include <VBox/vmm/em.h>
64# include <VBox/vmm/hm_svm.h>
65#endif
66#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
67# include <VBox/vmm/hmvmxinline.h>
68#endif
69#include <VBox/vmm/tm.h>
70#include <VBox/vmm/dbgf.h>
71#include <VBox/vmm/dbgftrace.h>
72#ifndef TST_IEM_CHECK_MC
73# include "IEMInternal.h"
74#endif
75#include <VBox/vmm/vmcc.h>
76#include <VBox/log.h>
77#include <VBox/err.h>
78#include <VBox/param.h>
79#include <VBox/dis.h>
80#include <VBox/disopcode-x86-amd64.h>
81#include <iprt/asm-math.h>
82#include <iprt/assert.h>
83#include <iprt/mem.h>
84#include <iprt/string.h>
85#include <iprt/x86.h>
86
87#ifndef TST_IEM_CHECK_MC
88# include "IEMInline.h"
89# include "IEMOpHlp.h"
90# include "IEMMc.h"
91#endif
92
93#include "IEMThreadedFunctions.h"
94
95
96/*
97 * Narrow down configs here to avoid wasting time on unused configs here.
98 */
99
100#ifndef IEM_WITH_CODE_TLB
101# error The code TLB must be enabled for the recompiler.
102#endif
103
104#ifndef IEM_WITH_DATA_TLB
105# error The data TLB must be enabled for the recompiler.
106#endif
107
108#ifndef IEM_WITH_SETJMP
109# error The setjmp approach must be enabled for the recompiler.
110#endif
111
112
113/*********************************************************************************************************************************
114* Structures and Typedefs *
115*********************************************************************************************************************************/
116/**
117 * A call for the threaded call table.
118 */
119typedef struct IEMTHRDEDCALLENTRY
120{
121 /** The function to call (IEMTHREADEDFUNCS). */
122 uint16_t enmFunction;
123 uint16_t uUnused0;
124
125 /** Offset into IEMTB::pabOpcodes. */
126 uint16_t offOpcode;
127 /** The opcode length. */
128 uint8_t cbOpcode;
129 uint8_t uUnused1;
130
131 /** Generic parameters. */
132 uint64_t auParams[3];
133} IEMTHRDEDCALLENTRY;
134AssertCompileSize(IEMTHRDEDCALLENTRY, sizeof(uint64_t) * 4);
135/** Pointer to a threaded call entry. */
136typedef IEMTHRDEDCALLENTRY *PIEMTHRDEDCALLENTRY;
137/** Pointer to a const threaded call entry. */
138typedef IEMTHRDEDCALLENTRY const *PCIEMTHRDEDCALLENTRY;
139
140
141
142/**
143 * Translation block.
144 */
145typedef struct IEMTB
146{
147 /** Next block with the same hash table entry. */
148 PIEMTB volatile pNext;
149 /** List on the local VCPU for blocks. */
150 RTLISTNODE LocalList;
151
152 /** @name What uniquely identifies the block.
153 * @{ */
154 RTGCPHYS GCPhysPc;
155 /** IEMTB_F_XXX (i.e. IEM_F_XXX ++). */
156 uint32_t fFlags;
157 union
158 {
159 struct
160 {
161 /**< Relevant CS X86DESCATTR_XXX bits. */
162 uint16_t fAttr;
163 } x86;
164 };
165 /** @} */
166
167 union
168 {
169 struct
170 {
171 /** The call sequence table. */
172 PIEMTHRDEDCALLENTRY paCalls;
173 /** Number of calls in paCalls. */
174 uint16_t cCalls;
175 /** Number of calls allocated. */
176 uint16_t cAllocated;
177 } Thrd;
178 };
179
180
181 /** Number of bytes of opcodes stored in pabOpcodes. */
182 uint16_t cbOpcodes;
183 /** The max storage available in the pabOpcodes block. */
184 uint16_t cbOpcodesAllocated;
185 /** Pointer to the opcode bytes this block was recompiled from. */
186 uint8_t *pabOpcodes;
187
188#if 0
189 struct
190 {
191 uint16_t offOpcodes;
192 uint16_t cbOpcodes;
193 } aRanges;
194
195 /** Physical pages that the . */
196 RTGCPHYS aGCPhysPgs[2];
197#endif
198
199} IEMTB;
200
201
202/*********************************************************************************************************************************
203* Internal Functions *
204*********************************************************************************************************************************/
205static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb);
206
207
208/*********************************************************************************************************************************
209* Defined Constants And Macros *
210*********************************************************************************************************************************/
211#define g_apfnOneByteMap g_apfnIemThreadedRecompilerOneByteMap
212
213
214#undef IEM_MC_CALC_RM_EFF_ADDR
215#ifndef IEM_WITH_SETJMP
216# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
217 uint64_t uEffAddrInfo; \
218 IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddrJmpEx(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff), &uEffAddrInfo))
219#else
220# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
221 uint64_t uEffAddrInfo; \
222 ((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmpEx(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &uEffAddrInfo))
223#endif
224
225#define IEM_MC2_BEGIN_EMIT_CALLS() \
226 { \
227 PIEMTB const pTb = pVCpu->iem.s.pCurTbR3; \
228 AssertMsg(pVCpu->iem.s.offOpcode == IEM_GET_INSTR_LEN(pVCpu), \
229 ("%u vs %u (%04x:%08RX64)\n", pVCpu->iem.s.offOpcode, IEM_GET_INSTR_LEN(pVCpu), \
230 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip)); \
231 /** @todo check for cross page stuff */ \
232 if (!(pTb->fFlags & IEMTB_F_CS_LIM_CHECKS)) \
233 { /* likely */ } \
234 else \
235 { \
236 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
237 pCall->enmFunction = kIemThreadedFunc_CheckCsLim; \
238 pCall->offOpcode = pTb->cbOpcodes; \
239 pCall->auParams[0] = pCall->cbOpcode = IEM_GET_INSTR_LEN(pVCpu); \
240 pCall->auParams[1] = 0; \
241 pCall->auParams[2] = 0; \
242 } \
243 do { } while (0)
244
245#define IEM_MC2_EMIT_CALL_0(a_enmFunction) do { \
246 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
247 \
248 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
249 pCall->enmFunction = a_enmFunction; \
250 pCall->offOpcode = pTb->cbOpcodes; \
251 pCall->cbOpcode = IEM_GET_INSTR_LEN(pVCpu); \
252 pCall->auParams[0] = 0; \
253 pCall->auParams[1] = 0; \
254 pCall->auParams[2] = 0; \
255 } while (0)
256#define IEM_MC2_EMIT_CALL_1(a_enmFunction, a_uArg0) do { \
257 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
258 uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
259 \
260 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
261 pCall->enmFunction = a_enmFunction; \
262 pCall->offOpcode = pTb->cbOpcodes; \
263 pCall->cbOpcode = IEM_GET_INSTR_LEN(pVCpu); \
264 pCall->auParams[0] = a_uArg0; \
265 pCall->auParams[1] = 0; \
266 pCall->auParams[2] = 0; \
267 } while (0)
268#define IEM_MC2_EMIT_CALL_2(a_enmFunction, a_uArg0, a_uArg1) do { \
269 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
270 uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
271 uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
272 \
273 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
274 pCall->enmFunction = a_enmFunction; \
275 pCall->offOpcode = pTb->cbOpcodes; \
276 pCall->cbOpcode = IEM_GET_INSTR_LEN(pVCpu); \
277 pCall->auParams[0] = a_uArg0; \
278 pCall->auParams[1] = a_uArg1; \
279 pCall->auParams[2] = 0; \
280 } while (0)
281#define IEM_MC2_EMIT_CALL_3(a_enmFunction, a_uArg0, a_uArg1, a_uArg2) do { \
282 IEMTHREADEDFUNCS const enmFunctionCheck = a_enmFunction; RT_NOREF(enmFunctionCheck); \
283 uint64_t const uArg0Check = (a_uArg0); RT_NOREF(uArg0Check); \
284 uint64_t const uArg1Check = (a_uArg1); RT_NOREF(uArg1Check); \
285 uint64_t const uArg2Check = (a_uArg2); RT_NOREF(uArg2Check); \
286 \
287 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++]; \
288 pCall->enmFunction = a_enmFunction; \
289 pCall->offOpcode = pTb->cbOpcodes; \
290 pCall->cbOpcode = IEM_GET_INSTR_LEN(pVCpu); \
291 pCall->auParams[0] = a_uArg0; \
292 pCall->auParams[1] = a_uArg1; \
293 pCall->auParams[2] = a_uArg2; \
294 } while (0)
295
296#define IEM_MC2_END_EMIT_CALLS() \
297 } while (0)
298
299
300/*
301 * IEM_MC_DEFER_TO_CIMPL_0 is easily wrapped up.
302 *
303 * Doing so will also take care of IEMOP_RAISE_DIVIDE_ERROR, IEMOP_RAISE_INVALID_LOCK_PREFIX,
304 * IEMOP_RAISE_INVALID_OPCODE and their users.
305 */
306#undef IEM_MC_DEFER_TO_CIMPL_0_RET
307#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_pfnCImpl) return iemThreadedRecompilerMcDeferToCImpl0(pVCpu, a_pfnCImpl)
308
309typedef IEM_CIMPL_DECL_TYPE_0(FNIEMCIMPL0);
310typedef FNIEMCIMPL0 *PFNIEMCIMPL0;
311
312DECLINLINE(VBOXSTRICTRC) iemThreadedRecompilerMcDeferToCImpl0(PVMCPUCC pVCpu, PFNIEMCIMPL0 pfnCImpl)
313{
314 return pfnCImpl(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
315}
316
317/**
318 * Calculates the effective address of a ModR/M memory operand, extended version
319 * for use in the recompilers.
320 *
321 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
322 *
323 * May longjmp on internal error.
324 *
325 * @return The effective address.
326 * @param pVCpu The cross context virtual CPU structure of the calling thread.
327 * @param bRm The ModRM byte.
328 * @param cbImmAndRspOffset - First byte: The size of any immediate
329 * following the effective address opcode bytes
330 * (only for RIP relative addressing).
331 * - Second byte: RSP displacement (for POP [ESP]).
332 * @param puInfo Extra info: 32-bit displacement (bits 31:0) and
333 * SIB byte (bits 39:32).
334 *
335 * @note This must be defined in a source file with matching
336 * IEM_WITH_CODE_TLB_AND_OPCODE_BUF define till the define is made default
337 * or implemented differently...
338 */
339RTGCPTR iemOpHlpCalcRmEffAddrJmpEx(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset, uint64_t *puInfo) IEM_NOEXCEPT_MAY_LONGJMP
340{
341 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
342# define SET_SS_DEF() \
343 do \
344 { \
345 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
346 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
347 } while (0)
348
349 if (!IEM_IS_64BIT_CODE(pVCpu))
350 {
351/** @todo Check the effective address size crap! */
352 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
353 {
354 uint16_t u16EffAddr;
355
356 /* Handle the disp16 form with no registers first. */
357 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
358 {
359 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
360 *puInfo = u16EffAddr;
361 }
362 else
363 {
364 /* Get the displacment. */
365 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
366 {
367 case 0: u16EffAddr = 0; break;
368 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
369 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
370 default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */
371 }
372 *puInfo = u16EffAddr;
373
374 /* Add the base and index registers to the disp. */
375 switch (bRm & X86_MODRM_RM_MASK)
376 {
377 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
378 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
379 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
380 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
381 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
382 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
383 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
384 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
385 }
386 }
387
388 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16 uInfo=%#RX64\n", u16EffAddr, *puInfo));
389 return u16EffAddr;
390 }
391
392 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
393 uint32_t u32EffAddr;
394 uint64_t uInfo;
395
396 /* Handle the disp32 form with no registers first. */
397 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
398 {
399 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
400 uInfo = u32EffAddr;
401 }
402 else
403 {
404 /* Get the register (or SIB) value. */
405 uInfo = 0;
406 switch ((bRm & X86_MODRM_RM_MASK))
407 {
408 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
409 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
410 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
411 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
412 case 4: /* SIB */
413 {
414 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
415 uInfo = (uint64_t)bSib << 32;
416
417 /* Get the index and scale it. */
418 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
419 {
420 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
421 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
422 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
423 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
424 case 4: u32EffAddr = 0; /*none */ break;
425 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
426 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
427 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
428 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
429 }
430 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
431
432 /* add base */
433 switch (bSib & X86_SIB_BASE_MASK)
434 {
435 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
436 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
437 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
438 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
439 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
440 case 5:
441 if ((bRm & X86_MODRM_MOD_MASK) != 0)
442 {
443 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
444 SET_SS_DEF();
445 }
446 else
447 {
448 uint32_t u32Disp;
449 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
450 u32EffAddr += u32Disp;
451 uInfo |= u32Disp;
452 }
453 break;
454 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
455 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
456 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
457 }
458 break;
459 }
460 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
461 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
462 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
463 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
464 }
465
466 /* Get and add the displacement. */
467 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
468 {
469 case 0:
470 break;
471 case 1:
472 {
473 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
474 u32EffAddr += i8Disp;
475 uInfo |= (uint32_t)(int32_t)i8Disp;
476 break;
477 }
478 case 2:
479 {
480 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
481 u32EffAddr += u32Disp;
482 uInfo |= u32Disp;
483 break;
484 }
485 default:
486 AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */
487 }
488 }
489
490 *puInfo = uInfo;
491 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32 uInfo=%#RX64\n", u32EffAddr, uInfo));
492 return u32EffAddr;
493 }
494
495 uint64_t u64EffAddr;
496 uint64_t uInfo;
497
498 /* Handle the rip+disp32 form with no registers first. */
499 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
500 {
501 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
502 uInfo = (uint32_t)u64EffAddr;
503 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
504 }
505 else
506 {
507 /* Get the register (or SIB) value. */
508 uInfo = 0;
509 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
510 {
511 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
512 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
513 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
514 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
515 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
516 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
517 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
518 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
519 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
520 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
521 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
522 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
523 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
524 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
525 /* SIB */
526 case 4:
527 case 12:
528 {
529 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
530 uInfo = (uint64_t)bSib << 32;
531
532 /* Get the index and scale it. */
533 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
534 {
535 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
536 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
537 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
538 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
539 case 4: u64EffAddr = 0; /*none */ break;
540 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
541 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
542 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
543 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
544 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
545 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
546 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
547 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
548 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
549 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
550 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
551 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
552 }
553 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
554
555 /* add base */
556 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
557 {
558 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
559 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
560 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
561 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
562 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
563 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
564 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
565 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
566 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
567 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
568 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
569 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
570 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
571 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
572 /* complicated encodings */
573 case 5:
574 case 13:
575 if ((bRm & X86_MODRM_MOD_MASK) != 0)
576 {
577 if (!pVCpu->iem.s.uRexB)
578 {
579 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
580 SET_SS_DEF();
581 }
582 else
583 u64EffAddr += pVCpu->cpum.GstCtx.r13;
584 }
585 else
586 {
587 uint32_t u32Disp;
588 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
589 u64EffAddr += (int32_t)u32Disp;
590 uInfo |= u32Disp;
591 }
592 break;
593 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
594 }
595 break;
596 }
597 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
598 }
599
600 /* Get and add the displacement. */
601 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
602 {
603 case 0:
604 break;
605 case 1:
606 {
607 int8_t i8Disp;
608 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
609 u64EffAddr += i8Disp;
610 uInfo |= (uint32_t)(int32_t)i8Disp;
611 break;
612 }
613 case 2:
614 {
615 uint32_t u32Disp;
616 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
617 u64EffAddr += (int32_t)u32Disp;
618 uInfo |= u32Disp;
619 break;
620 }
621 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
622 }
623
624 }
625
626 *puInfo = uInfo;
627 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
628 {
629 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr, uInfo));
630 return u64EffAddr;
631 }
632 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
633 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr & UINT32_MAX, uInfo));
634 return u64EffAddr & UINT32_MAX;
635}
636
637
638/*
639 * Include the "annotated" IEMAllInstructions*.cpp.h files.
640 */
641#include "IEMThreadedInstructions.cpp.h"
642
643/*
644 * Translation block management.
645 */
646typedef struct IEMTBCACHE
647{
648 uint32_t cHash;
649 uint32_t uHashMask;
650 PIEMTB apHash[_64K];
651} IEMTBCACHE;
652
653static IEMTBCACHE g_TbCache = { _64K, 0xffff, }; /**< Quick and dirty. */
654
655#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
656 ( ((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
657
658
659/**
660 * Allocate a translation block for threadeded recompilation.
661 *
662 * @returns Pointer to the translation block on success, NULL on failure.
663 * @param pVM The cross context virtual machine structure.
664 * @param pVCpu The cross context virtual CPU structure of the calling
665 * thread.
666 * @param GCPhysPc The physical address corresponding to RIP + CS.BASE.
667 * @param fExtraFlags Extra flags (IEMTB_F_XXX).
668 */
669static PIEMTB iemThreadedTbAlloc(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
670{
671 /*
672 * Just using the heap for now. Will make this more efficient and
673 * complicated later, don't worry. :-)
674 */
675 PIEMTB pTb = (PIEMTB)RTMemAlloc(sizeof(IEMTB));
676 if (pTb)
677 {
678 unsigned const cCalls = 128;
679 pTb->Thrd.paCalls = (PIEMTHRDEDCALLENTRY)RTMemAlloc(sizeof(IEMTHRDEDCALLENTRY) * cCalls);
680 if (pTb->Thrd.paCalls)
681 {
682 pTb->pabOpcodes = (uint8_t *)RTMemAlloc(cCalls * 16); /* This will be reallocated later. */
683 if (pTb->pabOpcodes)
684 {
685 pTb->Thrd.cAllocated = cCalls;
686 pTb->cbOpcodesAllocated = cCalls * 16;
687 pTb->Thrd.cCalls = 0;
688 pTb->cbOpcodes = 0;
689 pTb->pNext = NULL;
690 RTListInit(&pTb->LocalList);
691 pTb->GCPhysPc = GCPhysPc;
692 pTb->x86.fAttr = (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u;
693 pTb->fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags;
694 pVCpu->iem.s.cTbAllocs++;
695 return pTb;
696 }
697 RTMemFree(pTb->Thrd.paCalls);
698 }
699 RTMemFree(pTb);
700 }
701 RT_NOREF(pVM);
702 return NULL;
703}
704
705
706/**
707 * Frees pTb.
708 *
709 * @param pVM The cross context virtual machine structure.
710 * @param pVCpu The cross context virtual CPU structure of the calling
711 * thread.
712 * @param pTb The translation block to free..
713 */
714static void iemThreadedTbFree(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
715{
716 RT_NOREF(pVM);
717 AssertPtr(pTb);
718
719 AssertCompile(IEMTB_F_STATE_OBSOLETE == IEMTB_F_STATE_MASK);
720 pTb->fFlags |= IEMTB_F_STATE_OBSOLETE; /* works, both bits set */
721
722 /* Unlink it from the hash table: */
723 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
724 PIEMTB pTbCur = g_TbCache.apHash[idxHash];
725 if (pTbCur == pTb)
726 g_TbCache.apHash[idxHash] = pTb->pNext;
727 else
728 while (pTbCur)
729 {
730 PIEMTB const pNextTb = pTbCur->pNext;
731 if (pNextTb == pTb)
732 {
733 pTbCur->pNext = pTb->pNext;
734 break;
735 }
736 pTbCur = pNextTb;
737 }
738
739 /* Free it. */
740 RTMemFree(pTb->Thrd.paCalls);
741 pTb->Thrd.paCalls = NULL;
742
743 RTMemFree(pTb->pabOpcodes);
744 pTb->pabOpcodes = NULL;
745
746 RTMemFree(pTb);
747 pVCpu->iem.s.cTbFrees++;
748}
749
750
751static PIEMTB iemThreadedTbLookup(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
752{
753 uint32_t const fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags | IEMTB_F_STATE_READY;
754 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, fFlags, GCPhysPc);
755 Log10(("TB lookup: idxHash=%#x fFlags=%#x GCPhysPc=%RGp\n", idxHash, fFlags, GCPhysPc));
756 PIEMTB pTb = g_TbCache.apHash[idxHash];
757 while (pTb)
758 {
759 if (pTb->GCPhysPc == GCPhysPc)
760 {
761 if (pTb->fFlags == fFlags)
762 {
763 if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
764 {
765#ifdef VBOX_WITH_STATISTICS
766 pVCpu->iem.s.cTbLookupHits++;
767#endif
768 return pTb;
769 }
770 Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
771 }
772 else
773 Log11(("TB miss: fFlags: %#x, wanted %#x\n", pTb->fFlags, fFlags));
774 }
775 else
776 Log11(("TB miss: GCPhysPc: %#x, wanted %#x\n", pTb->GCPhysPc, GCPhysPc));
777
778 pTb = pTb->pNext;
779 }
780 RT_NOREF(pVM);
781 pVCpu->iem.s.cTbLookupMisses++;
782 return pTb;
783}
784
785
786static void iemThreadedTbAdd(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
787{
788 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
789 pTb->pNext = g_TbCache.apHash[idxHash];
790 g_TbCache.apHash[idxHash] = pTb;
791 Log12(("TB added: %p %RGp LB %#x fl=%#x idxHash=%#x\n", pTb, pTb->GCPhysPc, pTb->cbOpcodes, pTb->fFlags, idxHash));
792 RT_NOREF(pVM, pVCpu);
793}
794
795
796/*
797 * Real code.
798 */
799
800#ifdef LOG_ENABLED
801/**
802 * Logs the current instruction.
803 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
804 * @param pszFunction The IEM function doing the execution.
805 */
806static void iemThreadedLogCurInstr(PVMCPUCC pVCpu, const char *pszFunction) RT_NOEXCEPT
807{
808# ifdef IN_RING3
809 if (LogIs2Enabled())
810 {
811 char szInstr[256];
812 uint32_t cbInstr = 0;
813 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
814 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
815 szInstr, sizeof(szInstr), &cbInstr);
816
817 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
818 Log2(("**** %s fExec=%x pTb=%p\n"
819 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
820 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
821 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
822 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
823 " %s\n"
824 , pszFunction, pVCpu->iem.s.fExec, pVCpu->iem.s.pCurTbR3,
825 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
826 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
827 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
828 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
829 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
830 szInstr));
831
832 if (LogIs3Enabled())
833 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
834 }
835 else
836# endif
837 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
838 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
839}
840#endif /* LOG_ENABLED */
841
842
843static VBOXSTRICTRC iemThreadedCompileLongJumped(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
844{
845 RT_NOREF(pVM, pVCpu);
846 return rcStrict;
847}
848
849
850/**
851 * Initializes the decoder state when compiling TBs.
852 *
853 * This presumes that fExec has already be initialized.
854 *
855 * This is very similar to iemInitDecoder() and iemReInitDecoder(), so may need
856 * to apply fixes to them as well.
857 *
858 * @param pVCpu The cross context virtual CPU structure of the calling
859 * thread.
860 * @param fReInit Clear for the first call for a TB, set for subsequent calls
861 * from inside the compile loop where we can skip a couple of
862 * things.
863 */
864DECL_FORCE_INLINE(void) iemThreadedCompileInitDecoder(PVMCPUCC pVCpu, bool const fReInit)
865{
866 /* ASSUMES: That iemInitExec was already called and that anyone changing
867 CPU state affecting the fExec bits since then will have updated fExec! */
868 AssertMsg((pVCpu->iem.s.fExec & ~IEM_F_USER_OPTS) == iemCalcExecFlags(pVCpu),
869 ("fExec=%#x iemCalcExecModeFlags=%#x\n", pVCpu->iem.s.fExec, iemCalcExecFlags(pVCpu)));
870
871 IEMMODE const enmMode = IEM_GET_CPU_MODE(pVCpu);
872
873 /* Decoder state: */
874 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
875 pVCpu->iem.s.enmEffAddrMode = enmMode;
876 if (enmMode != IEMMODE_64BIT)
877 {
878 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
879 pVCpu->iem.s.enmEffOpSize = enmMode;
880 }
881 else
882 {
883 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
884 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
885 }
886 pVCpu->iem.s.fPrefixes = 0;
887 pVCpu->iem.s.uRexReg = 0;
888 pVCpu->iem.s.uRexB = 0;
889 pVCpu->iem.s.uRexIndex = 0;
890 pVCpu->iem.s.idxPrefix = 0;
891 pVCpu->iem.s.uVex3rdReg = 0;
892 pVCpu->iem.s.uVexLength = 0;
893 pVCpu->iem.s.fEvexStuff = 0;
894 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
895 pVCpu->iem.s.offModRm = 0;
896 pVCpu->iem.s.iNextMapping = 0;
897
898 if (!fReInit)
899 {
900 pVCpu->iem.s.cActiveMappings = 0;
901 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
902 pVCpu->iem.s.fEndTb = false;
903 }
904 else
905 {
906 Assert(pVCpu->iem.s.cActiveMappings == 0);
907 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
908 Assert(pVCpu->iem.s.fEndTb == false);
909 }
910
911#ifdef DBGFTRACE_ENABLED
912 switch (IEM_GET_CPU_MODE(pVCpu))
913 {
914 case IEMMODE_64BIT:
915 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);
916 break;
917 case IEMMODE_32BIT:
918 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
919 break;
920 case IEMMODE_16BIT:
921 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
922 break;
923 }
924#endif
925}
926
927
928/**
929 * Initializes the opcode fetcher when starting the compilation.
930 *
931 * @param pVCpu The cross context virtual CPU structure of the calling
932 * thread.
933 */
934DECL_FORCE_INLINE(void) iemThreadedCompileInitOpcodeFetching(PVMCPUCC pVCpu)
935{
936 /* Almost everything is done by iemGetPcWithPhysAndCode() already. We just need to initialize the index into abOpcode. */
937#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
938 pVCpu->iem.s.offOpcode = 0;
939#else
940 RT_NOREF(pVCpu);
941#endif
942}
943
944
945/**
946 * Re-initializes the opcode fetcher between instructions while compiling.
947 *
948 * @param pVCpu The cross context virtual CPU structure of the calling
949 * thread.
950 */
951DECL_FORCE_INLINE(void) iemThreadedCompileReInitOpcodeFetching(PVMCPUCC pVCpu)
952{
953 if (pVCpu->iem.s.pbInstrBuf)
954 {
955 uint64_t off = pVCpu->cpum.GstCtx.rip;
956 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
957 off += pVCpu->cpum.GstCtx.cs.u64Base;
958 off -= pVCpu->iem.s.uInstrBufPc;
959 if (off < pVCpu->iem.s.cbInstrBufTotal)
960 {
961 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
962 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
963 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
964 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
965 else
966 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
967 }
968 else
969 {
970 pVCpu->iem.s.pbInstrBuf = NULL;
971 pVCpu->iem.s.offInstrNextByte = 0;
972 pVCpu->iem.s.offCurInstrStart = 0;
973 pVCpu->iem.s.cbInstrBuf = 0;
974 pVCpu->iem.s.cbInstrBufTotal = 0;
975 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
976 }
977 }
978 else
979 {
980 pVCpu->iem.s.offInstrNextByte = 0;
981 pVCpu->iem.s.offCurInstrStart = 0;
982 pVCpu->iem.s.cbInstrBuf = 0;
983 pVCpu->iem.s.cbInstrBufTotal = 0;
984#ifdef VBOX_STRICT
985 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
986#endif
987 }
988#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
989 pVCpu->iem.s.offOpcode = 0;
990#endif
991}
992
993
994/**
995 * Compiles a new TB and executes it.
996 *
997 * We combine compilation and execution here as it makes it simpler code flow
998 * in the main loop and it allows interpreting while compiling if we want to
999 * explore that option.
1000 *
1001 * @returns Strict VBox status code.
1002 * @param pVM The cross context virtual machine structure.
1003 * @param pVCpu The cross context virtual CPU structure of the calling
1004 * thread.
1005 * @param GCPhysPc The physical address corresponding to the current
1006 * RIP+CS.BASE.
1007 * @param fExtraFlags Extra translation block flags: IEMTB_F_TYPE_THREADED and
1008 * maybe IEMTB_F_RIP_CHECKS.
1009 */
1010static VBOXSTRICTRC iemThreadedCompile(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
1011{
1012 /*
1013 * Allocate a new translation block.
1014 */
1015 PIEMTB pTb = iemThreadedTbAlloc(pVM, pVCpu, GCPhysPc, fExtraFlags | IEMTB_F_STATE_COMPILING);
1016 AssertReturn(pTb, VERR_IEM_TB_ALLOC_FAILED);
1017
1018 /* Set the current TB so iemThreadedCompileLongJumped and the CIMPL
1019 functions may get at it. */
1020 pVCpu->iem.s.pCurTbR3 = pTb;
1021
1022 /*
1023 * Now for the recomplication. (This mimicks IEMExecLots in many ways.)
1024 */
1025 iemThreadedCompileInitDecoder(pVCpu, false /*fReInit*/);
1026 iemThreadedCompileInitOpcodeFetching(pVCpu);
1027 VBOXSTRICTRC rcStrict;
1028 for (;;)
1029 {
1030 /* Process the next instruction. */
1031#ifdef LOG_ENABLED
1032 iemThreadedLogCurInstr(pVCpu, "CC");
1033 uint16_t const uCsLog = pVCpu->cpum.GstCtx.cs.Sel;
1034 uint64_t const uRipLog = pVCpu->cpum.GstCtx.rip;
1035#endif
1036 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
1037 uint16_t const cCallsPrev = pTb->Thrd.cCalls;
1038
1039 rcStrict = FNIEMOP_CALL(g_apfnIemThreadedRecompilerOneByteMap[b]);
1040 if ( rcStrict == VINF_SUCCESS
1041 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS
1042 && !pVCpu->iem.s.fEndTb)
1043 {
1044 Assert(pTb->Thrd.cCalls > cCallsPrev);
1045 Assert(cCallsPrev - pTb->Thrd.cCalls < 5);
1046
1047 memcpy(&pTb->pabOpcodes[pTb->cbOpcodes], pVCpu->iem.s.abOpcode, pVCpu->iem.s.offOpcode);
1048 pTb->cbOpcodes += pVCpu->iem.s.offOpcode;
1049 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1050 pVCpu->iem.s.cInstructions++;
1051 }
1052 else if (pTb->Thrd.cCalls > 0)
1053 {
1054 Log8(("%04x:%08RX64: End TB - %u calls, rc=%d\n", uCsLog, uRipLog, pTb->Thrd.cCalls, VBOXSTRICTRC_VAL(rcStrict)));
1055
1056 if (cCallsPrev != pTb->Thrd.cCalls)
1057 {
1058 memcpy(&pTb->pabOpcodes[pTb->cbOpcodes], pVCpu->iem.s.abOpcode, pVCpu->iem.s.offOpcode);
1059 pTb->cbOpcodes += pVCpu->iem.s.offOpcode;
1060 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1061 pVCpu->iem.s.cInstructions++;
1062 }
1063 break;
1064 }
1065 else
1066 {
1067 Log8(("%04x:%08RX64: End TB - 0 calls, rc=%d\n", uCsLog, uRipLog, VBOXSTRICTRC_VAL(rcStrict)));
1068 pVCpu->iem.s.pCurTbR3 = NULL;
1069 iemThreadedTbFree(pVM, pVCpu, pTb);
1070 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1071 }
1072
1073 /* Still space in the TB? */
1074 if (pTb->Thrd.cCalls + 5 < pTb->Thrd.cAllocated)
1075 iemThreadedCompileInitDecoder(pVCpu, true /*fReInit*/);
1076 else
1077 {
1078 Log8(("%04x:%08RX64: End TB - %u calls - full\n", uCsLog, uRipLog, pTb->Thrd.cCalls));
1079 break;
1080 }
1081 iemThreadedCompileReInitOpcodeFetching(pVCpu);
1082 }
1083
1084 /*
1085 * Complete the TB and link it.
1086 */
1087 pTb->fFlags = (pTb->fFlags & ~IEMTB_F_STATE_MASK) | IEMTB_F_STATE_READY;
1088 iemThreadedTbAdd(pVM, pVCpu, pTb);
1089
1090#ifdef IEM_COMPILE_ONLY_MODE
1091 /*
1092 * Execute the translation block.
1093 */
1094#endif
1095
1096 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1097}
1098
1099
1100/**
1101 * Executes a translation block.
1102 *
1103 * @returns Strict VBox status code.
1104 * @param pVCpu The cross context virtual CPU structure of the calling
1105 * thread.
1106 * @param pTb The translation block to execute.
1107 */
1108static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb)
1109{
1110 /* Check the opcodes in the first page before starting execution. */
1111 uint32_t const cbLeadOpcodes = RT_MIN(pTb->cbOpcodes, pVCpu->iem.s.cbInstrBufTotal - pVCpu->iem.s.offInstrNextByte);
1112 if (memcmp(pTb->pabOpcodes, &pVCpu->iem.s.pbInstrBuf[pVCpu->iem.s.offInstrNextByte], cbLeadOpcodes) == 0)
1113 Assert( pTb->cbOpcodes == cbLeadOpcodes
1114 || cbLeadOpcodes == (GUEST_PAGE_SIZE - (pTb->GCPhysPc & GUEST_PAGE_OFFSET_MASK)));
1115 else
1116 {
1117 Log11(("TB obsolete: %p GCPhys=%RGp\n", pTb, pTb->GCPhysPc));
1118 iemThreadedTbFree(pVCpu->pVMR3, pVCpu, pTb);
1119 return VINF_SUCCESS;
1120 }
1121
1122 /* Set the current TB so CIMPL function may get at it. */
1123 pVCpu->iem.s.pCurTbR3 = pTb;
1124 pVCpu->iem.s.cTbExec++;
1125
1126 /* The execution loop. */
1127 PCIEMTHRDEDCALLENTRY pCallEntry = pTb->Thrd.paCalls;
1128 uint32_t cCallsLeft = pTb->Thrd.cCalls;
1129 while (cCallsLeft-- > 0)
1130 {
1131#ifdef LOG_ENABLED
1132 iemThreadedLogCurInstr(pVCpu, "EX");
1133 Log9(("%04x:%08RX64: #%d - %d %s\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
1134 pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->enmFunction, g_apszIemThreadedFunctions[pCallEntry->enmFunction]));
1135#endif
1136 VBOXSTRICTRC const rcStrict = g_apfnIemThreadedFunctions[pCallEntry->enmFunction](pVCpu,
1137 pCallEntry->auParams[0],
1138 pCallEntry->auParams[1],
1139 pCallEntry->auParams[2]);
1140
1141 if (RT_LIKELY( rcStrict == VINF_SUCCESS
1142 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
1143 pCallEntry++;
1144 else
1145 {
1146 pVCpu->iem.s.pCurTbR3 = NULL;
1147
1148 /* Some status codes are just to get us out of this loop and
1149 continue in a different translation block. */
1150 if (rcStrict == VINF_IEM_REEXEC_MODE_CHANGED)
1151 return iemExecStatusCodeFiddling(pVCpu, VINF_SUCCESS);
1152 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1153 }
1154 }
1155
1156 pVCpu->iem.s.pCurTbR3 = NULL;
1157 return VINF_SUCCESS;
1158}
1159
1160
1161/**
1162 * This is called when the PC doesn't match the current pbInstrBuf.
1163 *
1164 * Upon return, we're ready for opcode fetching. But please note that
1165 * pbInstrBuf can be NULL iff the memory doesn't have readable backing (i.e.
1166 * MMIO or unassigned).
1167 */
1168static RTGCPHYS iemGetPcWithPhysAndCodeMissed(PVMCPUCC pVCpu)
1169{
1170 pVCpu->iem.s.pbInstrBuf = NULL;
1171 pVCpu->iem.s.offCurInstrStart = 0;
1172 pVCpu->iem.s.offInstrNextByte = 0;
1173 iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
1174 return pVCpu->iem.s.GCPhysInstrBuf + pVCpu->iem.s.offCurInstrStart;
1175}
1176
1177
1178/** @todo need private inline decl for throw/nothrow matching IEM_WITH_SETJMP? */
1179DECL_FORCE_INLINE_THROW(RTGCPHYS) iemGetPcWithPhysAndCode(PVMCPUCC pVCpu)
1180{
1181 /*
1182 * Set uCurTbStartPc to RIP and calc the effective PC.
1183 */
1184 uint64_t uPc = pVCpu->cpum.GstCtx.rip;
1185 pVCpu->iem.s.uCurTbStartPc = uPc;
1186 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
1187 uPc += pVCpu->cpum.GstCtx.cs.u64Base;
1188
1189 /*
1190 * Advance within the current buffer (PAGE) when possible.
1191 */
1192 if (pVCpu->iem.s.pbInstrBuf)
1193 {
1194 uint64_t off = uPc - pVCpu->iem.s.uInstrBufPc;
1195 if (off < pVCpu->iem.s.cbInstrBufTotal)
1196 {
1197 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1198 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1199 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1200 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1201 else
1202 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1203
1204 return pVCpu->iem.s.GCPhysInstrBuf + off;
1205 }
1206 }
1207 return iemGetPcWithPhysAndCodeMissed(pVCpu);
1208}
1209
1210
1211/**
1212 * Determines the extra IEMTB_F_XXX flags.
1213 *
1214 * @returns IEMTB_F_TYPE_THREADED and maybe IEMTB_F_RIP_CHECKS.
1215 * @param pVCpu The cross context virtual CPU structure of the calling
1216 * thread.
1217 */
1218DECL_FORCE_INLINE(uint32_t) iemGetTbFlagsForCurrentPc(PVMCPUCC pVCpu)
1219{
1220 /*
1221 * Return IEMTB_F_RIP_CHECKS if the current PC is invalid or if it is
1222 * likely to go invalid before the end of the translation block.
1223 */
1224 if (IEM_IS_64BIT_CODE(pVCpu))
1225 return IEMTB_F_TYPE_THREADED;
1226
1227 if (RT_LIKELY( pVCpu->cpum.GstCtx.eip < pVCpu->cpum.GstCtx.cs.u32Limit
1228 && pVCpu->cpum.GstCtx.eip - pVCpu->cpum.GstCtx.cs.u32Limit >= X86_PAGE_SIZE))
1229 return IEMTB_F_TYPE_THREADED;
1230
1231 return IEMTB_F_TYPE_THREADED | IEMTB_F_CS_LIM_CHECKS;
1232}
1233
1234
1235VMMDECL(VBOXSTRICTRC) IEMExecRecompilerThreaded(PVMCC pVM, PVMCPUCC pVCpu)
1236{
1237 /*
1238 * See if there is an interrupt pending in TRPM, inject it if we can.
1239 */
1240 if (!TRPMHasTrap(pVCpu))
1241 { /* likely */ }
1242 else
1243 {
1244 VBOXSTRICTRC rcStrict = iemExecInjectPendingTrap(pVCpu);
1245 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1246 { /*likely */ }
1247 else
1248 return rcStrict;
1249 }
1250
1251 /*
1252 * Init the execution environment.
1253 */
1254 iemInitExec(pVCpu, 0 /*fExecOpts*/);
1255
1256 /*
1257 * Run-loop.
1258 *
1259 * If we're using setjmp/longjmp we combine all the catching here to avoid
1260 * having to call setjmp for each block we're executing.
1261 */
1262 for (;;)
1263 {
1264 PIEMTB pTb = NULL;
1265 VBOXSTRICTRC rcStrict;
1266 IEM_TRY_SETJMP(pVCpu, rcStrict)
1267 {
1268 uint32_t const cPollRate = 511; /* EM.cpp passes 4095 to IEMExecLots, so an eigth of that seems reasonable for now. */
1269 for (uint32_t iIterations = 0; ; iIterations++)
1270 {
1271 /* Translate PC to physical address, we'll need this for both lookup and compilation. */
1272 RTGCPHYS const GCPhysPc = iemGetPcWithPhysAndCode(pVCpu);
1273 uint32_t const fExtraFlags = iemGetTbFlagsForCurrentPc(pVCpu);
1274
1275 pTb = iemThreadedTbLookup(pVM, pVCpu, GCPhysPc, fExtraFlags);
1276 if (pTb)
1277 rcStrict = iemThreadedTbExec(pVCpu, pTb);
1278 else
1279 rcStrict = iemThreadedCompile(pVM, pVCpu, GCPhysPc, fExtraFlags);
1280 if (rcStrict == VINF_SUCCESS)
1281 {
1282 Assert(pVCpu->iem.s.cActiveMappings == 0);
1283
1284 uint64_t fCpu = pVCpu->fLocalForcedActions;
1285 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
1286 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
1287 | VMCPU_FF_TLB_FLUSH
1288 | VMCPU_FF_UNHALT );
1289 if (RT_LIKELY( ( !fCpu
1290 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
1291 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
1292 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
1293 {
1294 if (RT_LIKELY( (iIterations & cPollRate) != 0
1295 || !TMTimerPollBool(pVM, pVCpu)))
1296 {
1297
1298 }
1299 else
1300 return VINF_SUCCESS;
1301 }
1302 else
1303 return VINF_SUCCESS;
1304 }
1305 else
1306 return rcStrict;
1307 }
1308 }
1309 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
1310 {
1311 pVCpu->iem.s.cLongJumps++;
1312 if (pVCpu->iem.s.cActiveMappings > 0)
1313 iemMemRollback(pVCpu);
1314
1315 /* If pTb isn't NULL we're in iemThreadedTbExec. */
1316 if (!pTb)
1317 {
1318 /* If pCurTbR3 is NULL, we're in iemGetPcWithPhysAndCode.*/
1319 pTb = pVCpu->iem.s.pCurTbR3;
1320 if (pTb)
1321 {
1322 /* If the pCurTbR3 block is in compiling state, we're in iemThreadedCompile,
1323 otherwise it's iemThreadedTbExec inside iemThreadedCompile (compile option). */
1324 if ((pTb->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_COMPILING)
1325 return iemThreadedCompileLongJumped(pVM, pVCpu, rcStrict);
1326 }
1327 }
1328 return rcStrict;
1329 }
1330 IEM_CATCH_LONGJMP_END(pVCpu);
1331 }
1332}
1333
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette