VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp@ 100761

Last change on this file since 100761 was 100761, checked in by vboxsync, 19 months ago

VMM/IEM: Check for IRQs every so often, especially after sti, popf and iret. Increased the hash table size. Disabled some debug code. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 61.5 KB
Line 
1/* $Id: IEMAllThrdRecompiler.cpp 100761 2023-08-01 02:24:11Z vboxsync $ */
2/** @file
3 * IEM - Instruction Decoding and Threaded Recompilation.
4 *
5 * Logging group IEM_RE_THREADED assignments:
6 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events. [same as IEM]
7 * - Flow (LogFlow) :
8 * - Level 2 (Log2) : Basic instruction execution state info. [same as IEM]
9 * - Level 3 (Log3) : More detailed execution state info. [same as IEM]
10 * - Level 4 (Log4) : Decoding mnemonics w/ EIP. [same as IEM]
11 * - Level 5 (Log5) : Decoding details. [same as IEM]
12 * - Level 6 (Log6) :
13 * - Level 7 (Log7) : TB obsoletion.
14 * - Level 8 (Log8) : TB compilation.
15 * - Level 9 (Log9) : TB exec.
16 * - Level 10 (Log10): TB block lookup.
17 * - Level 11 (Log11): TB block lookup details.
18 * - Level 12 (Log12): TB insertion.
19 */
20
21/*
22 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
23 *
24 * This file is part of VirtualBox base platform packages, as
25 * available from https://www.virtualbox.org.
26 *
27 * This program is free software; you can redistribute it and/or
28 * modify it under the terms of the GNU General Public License
29 * as published by the Free Software Foundation, in version 3 of the
30 * License.
31 *
32 * This program is distributed in the hope that it will be useful, but
33 * WITHOUT ANY WARRANTY; without even the implied warranty of
34 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
35 * General Public License for more details.
36 *
37 * You should have received a copy of the GNU General Public License
38 * along with this program; if not, see <https://www.gnu.org/licenses>.
39 *
40 * SPDX-License-Identifier: GPL-3.0-only
41 */
42
43
44/*********************************************************************************************************************************
45* Header Files *
46*********************************************************************************************************************************/
47#ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */
48# define LOG_GROUP LOG_GROUP_IEM_RE_THREADED
49#endif
50#define IEM_WITH_CODE_TLB_AND_OPCODE_BUF /* A bit hackish, but its all in IEMInline.h. */
51#define VMCPU_INCL_CPUM_GST_CTX
52#include <VBox/vmm/iem.h>
53#include <VBox/vmm/cpum.h>
54#include <VBox/vmm/apic.h>
55#include <VBox/vmm/pdm.h>
56#include <VBox/vmm/pgm.h>
57#include <VBox/vmm/iom.h>
58#include <VBox/vmm/em.h>
59#include <VBox/vmm/hm.h>
60#include <VBox/vmm/nem.h>
61#include <VBox/vmm/gim.h>
62#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
63# include <VBox/vmm/em.h>
64# include <VBox/vmm/hm_svm.h>
65#endif
66#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
67# include <VBox/vmm/hmvmxinline.h>
68#endif
69#include <VBox/vmm/tm.h>
70#include <VBox/vmm/dbgf.h>
71#include <VBox/vmm/dbgftrace.h>
72#ifndef TST_IEM_CHECK_MC
73# include "IEMInternal.h"
74#endif
75#include <VBox/vmm/vmcc.h>
76#include <VBox/log.h>
77#include <VBox/err.h>
78#include <VBox/param.h>
79#include <VBox/dis.h>
80#include <VBox/disopcode-x86-amd64.h>
81#include <iprt/asm-math.h>
82#include <iprt/assert.h>
83#include <iprt/mem.h>
84#include <iprt/string.h>
85#include <iprt/x86.h>
86
87#ifndef TST_IEM_CHECK_MC
88# include "IEMInline.h"
89# include "IEMOpHlp.h"
90# include "IEMMc.h"
91#endif
92
93#include "IEMThreadedFunctions.h"
94
95
96/*
97 * Narrow down configs here to avoid wasting time on unused configs here.
98 */
99
100#ifndef IEM_WITH_CODE_TLB
101# error The code TLB must be enabled for the recompiler.
102#endif
103
104#ifndef IEM_WITH_DATA_TLB
105# error The data TLB must be enabled for the recompiler.
106#endif
107
108#ifndef IEM_WITH_SETJMP
109# error The setjmp approach must be enabled for the recompiler.
110#endif
111
112
113/*********************************************************************************************************************************
114* Internal Functions *
115*********************************************************************************************************************************/
116static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb);
117
118
119/**
120 * Calculates the effective address of a ModR/M memory operand, extended version
121 * for use in the recompilers.
122 *
123 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
124 *
125 * May longjmp on internal error.
126 *
127 * @return The effective address.
128 * @param pVCpu The cross context virtual CPU structure of the calling thread.
129 * @param bRm The ModRM byte.
130 * @param cbImmAndRspOffset - First byte: The size of any immediate
131 * following the effective address opcode bytes
132 * (only for RIP relative addressing).
133 * - Second byte: RSP displacement (for POP [ESP]).
134 * @param puInfo Extra info: 32-bit displacement (bits 31:0) and
135 * SIB byte (bits 39:32).
136 *
137 * @note This must be defined in a source file with matching
138 * IEM_WITH_CODE_TLB_AND_OPCODE_BUF define till the define is made default
139 * or implemented differently...
140 */
141RTGCPTR iemOpHlpCalcRmEffAddrJmpEx(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset, uint64_t *puInfo) IEM_NOEXCEPT_MAY_LONGJMP
142{
143 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
144# define SET_SS_DEF() \
145 do \
146 { \
147 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
148 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
149 } while (0)
150
151 if (!IEM_IS_64BIT_CODE(pVCpu))
152 {
153/** @todo Check the effective address size crap! */
154 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
155 {
156 uint16_t u16EffAddr;
157
158 /* Handle the disp16 form with no registers first. */
159 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
160 {
161 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
162 *puInfo = u16EffAddr;
163 }
164 else
165 {
166 /* Get the displacment. */
167 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
168 {
169 case 0: u16EffAddr = 0; break;
170 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
171 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
172 default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */
173 }
174 *puInfo = u16EffAddr;
175
176 /* Add the base and index registers to the disp. */
177 switch (bRm & X86_MODRM_RM_MASK)
178 {
179 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
180 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
181 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
182 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
183 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
184 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
185 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
186 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
187 }
188 }
189
190 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16 uInfo=%#RX64\n", u16EffAddr, *puInfo));
191 return u16EffAddr;
192 }
193
194 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
195 uint32_t u32EffAddr;
196 uint64_t uInfo;
197
198 /* Handle the disp32 form with no registers first. */
199 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
200 {
201 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
202 uInfo = u32EffAddr;
203 }
204 else
205 {
206 /* Get the register (or SIB) value. */
207 uInfo = 0;
208 switch ((bRm & X86_MODRM_RM_MASK))
209 {
210 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
211 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
212 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
213 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
214 case 4: /* SIB */
215 {
216 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
217 uInfo = (uint64_t)bSib << 32;
218
219 /* Get the index and scale it. */
220 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
221 {
222 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
223 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
224 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
225 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
226 case 4: u32EffAddr = 0; /*none */ break;
227 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
228 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
229 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
230 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
231 }
232 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
233
234 /* add base */
235 switch (bSib & X86_SIB_BASE_MASK)
236 {
237 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
238 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
239 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
240 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
241 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
242 case 5:
243 if ((bRm & X86_MODRM_MOD_MASK) != 0)
244 {
245 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
246 SET_SS_DEF();
247 }
248 else
249 {
250 uint32_t u32Disp;
251 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
252 u32EffAddr += u32Disp;
253 uInfo |= u32Disp;
254 }
255 break;
256 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
257 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
258 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
259 }
260 break;
261 }
262 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
263 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
264 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
265 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
266 }
267
268 /* Get and add the displacement. */
269 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
270 {
271 case 0:
272 break;
273 case 1:
274 {
275 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
276 u32EffAddr += i8Disp;
277 uInfo |= (uint32_t)(int32_t)i8Disp;
278 break;
279 }
280 case 2:
281 {
282 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
283 u32EffAddr += u32Disp;
284 uInfo |= u32Disp;
285 break;
286 }
287 default:
288 AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */
289 }
290 }
291
292 *puInfo = uInfo;
293 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32 uInfo=%#RX64\n", u32EffAddr, uInfo));
294 return u32EffAddr;
295 }
296
297 uint64_t u64EffAddr;
298 uint64_t uInfo;
299
300 /* Handle the rip+disp32 form with no registers first. */
301 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
302 {
303 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
304 uInfo = (uint32_t)u64EffAddr;
305 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
306 }
307 else
308 {
309 /* Get the register (or SIB) value. */
310 uInfo = 0;
311 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
312 {
313 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
314 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
315 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
316 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
317 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
318 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
319 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
320 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
321 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
322 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
323 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
324 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
325 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
326 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
327 /* SIB */
328 case 4:
329 case 12:
330 {
331 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
332 uInfo = (uint64_t)bSib << 32;
333
334 /* Get the index and scale it. */
335 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
336 {
337 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
338 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
339 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
340 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
341 case 4: u64EffAddr = 0; /*none */ break;
342 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
343 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
344 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
345 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
346 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
347 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
348 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
349 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
350 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
351 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
352 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
353 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
354 }
355 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
356
357 /* add base */
358 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
359 {
360 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
361 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
362 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
363 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
364 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
365 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
366 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
367 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
368 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
369 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
370 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
371 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
372 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
373 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
374 /* complicated encodings */
375 case 5:
376 case 13:
377 if ((bRm & X86_MODRM_MOD_MASK) != 0)
378 {
379 if (!pVCpu->iem.s.uRexB)
380 {
381 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
382 SET_SS_DEF();
383 }
384 else
385 u64EffAddr += pVCpu->cpum.GstCtx.r13;
386 }
387 else
388 {
389 uint32_t u32Disp;
390 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
391 u64EffAddr += (int32_t)u32Disp;
392 uInfo |= u32Disp;
393 }
394 break;
395 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
396 }
397 break;
398 }
399 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
400 }
401
402 /* Get and add the displacement. */
403 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
404 {
405 case 0:
406 break;
407 case 1:
408 {
409 int8_t i8Disp;
410 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
411 u64EffAddr += i8Disp;
412 uInfo |= (uint32_t)(int32_t)i8Disp;
413 break;
414 }
415 case 2:
416 {
417 uint32_t u32Disp;
418 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
419 u64EffAddr += (int32_t)u32Disp;
420 uInfo |= u32Disp;
421 break;
422 }
423 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
424 }
425
426 }
427
428 *puInfo = uInfo;
429 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
430 {
431 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr, uInfo));
432 return u64EffAddr;
433 }
434 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
435 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr & UINT32_MAX, uInfo));
436 return u64EffAddr & UINT32_MAX;
437}
438
439
440/*
441 * Translation block management.
442 */
443
444typedef struct IEMTBCACHE
445{
446 uint32_t cHash;
447 uint32_t uHashMask;
448 PIEMTB apHash[_1M];
449} IEMTBCACHE;
450
451static IEMTBCACHE g_TbCache = { _1M, _1M - 1, }; /**< Quick and dirty. */
452
453#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
454 ( ((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
455
456
457/**
458 * Allocate a translation block for threadeded recompilation.
459 *
460 * @returns Pointer to the translation block on success, NULL on failure.
461 * @param pVM The cross context virtual machine structure.
462 * @param pVCpu The cross context virtual CPU structure of the calling
463 * thread.
464 * @param GCPhysPc The physical address corresponding to RIP + CS.BASE.
465 * @param fExtraFlags Extra flags (IEMTB_F_XXX).
466 */
467static PIEMTB iemThreadedTbAlloc(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
468{
469 /*
470 * Just using the heap for now. Will make this more efficient and
471 * complicated later, don't worry. :-)
472 */
473 PIEMTB pTb = (PIEMTB)RTMemAlloc(sizeof(IEMTB));
474 if (pTb)
475 {
476 unsigned const cCalls = 128;
477 pTb->Thrd.paCalls = (PIEMTHRDEDCALLENTRY)RTMemAlloc(sizeof(IEMTHRDEDCALLENTRY) * cCalls);
478 if (pTb->Thrd.paCalls)
479 {
480 pTb->pabOpcodes = (uint8_t *)RTMemAlloc(cCalls * 16); /* This will be reallocated later. */
481 if (pTb->pabOpcodes)
482 {
483 pTb->Thrd.cAllocated = cCalls;
484 pTb->cbOpcodesAllocated = cCalls * 16;
485 pTb->Thrd.cCalls = 0;
486 pTb->cbOpcodes = 0;
487 pTb->pNext = NULL;
488 RTListInit(&pTb->LocalList);
489 pTb->GCPhysPc = GCPhysPc;
490 pTb->x86.fAttr = (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u;
491 pTb->fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags;
492 pTb->cInstructions = 0;
493
494 /* Init the first opcode range. */
495 pTb->cRanges = 1;
496 pTb->aRanges[0].cbOpcodes = 0;
497 pTb->aRanges[0].offOpcodes = 0;
498 pTb->aRanges[0].offPhysPage = GCPhysPc & GUEST_PAGE_OFFSET_MASK;
499 pTb->aRanges[0].u2Unused = 0;
500 pTb->aRanges[0].idxPhysPage = 0;
501 pTb->aGCPhysPages[0] = NIL_RTGCPHYS;
502 pTb->aGCPhysPages[1] = NIL_RTGCPHYS;
503
504 pVCpu->iem.s.cTbAllocs++;
505 return pTb;
506 }
507 RTMemFree(pTb->Thrd.paCalls);
508 }
509 RTMemFree(pTb);
510 }
511 RT_NOREF(pVM);
512 return NULL;
513}
514
515
516/**
517 * Frees pTb.
518 *
519 * @param pVM The cross context virtual machine structure.
520 * @param pVCpu The cross context virtual CPU structure of the calling
521 * thread.
522 * @param pTb The translation block to free..
523 */
524static void iemThreadedTbFree(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
525{
526 RT_NOREF(pVM);
527 AssertPtr(pTb);
528
529 AssertCompile(IEMTB_F_STATE_OBSOLETE == IEMTB_F_STATE_MASK);
530 pTb->fFlags |= IEMTB_F_STATE_OBSOLETE; /* works, both bits set */
531
532 /* Unlink it from the hash table: */
533 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
534 PIEMTB pTbCur = g_TbCache.apHash[idxHash];
535 if (pTbCur == pTb)
536 g_TbCache.apHash[idxHash] = pTb->pNext;
537 else
538 while (pTbCur)
539 {
540 PIEMTB const pNextTb = pTbCur->pNext;
541 if (pNextTb == pTb)
542 {
543 pTbCur->pNext = pTb->pNext;
544 break;
545 }
546 pTbCur = pNextTb;
547 }
548
549 /* Free it. */
550 RTMemFree(pTb->Thrd.paCalls);
551 pTb->Thrd.paCalls = NULL;
552
553 RTMemFree(pTb->pabOpcodes);
554 pTb->pabOpcodes = NULL;
555
556 RTMemFree(pTb);
557 pVCpu->iem.s.cTbFrees++;
558}
559
560
561/**
562 * Called by opcode verifier functions when they detect a problem.
563 */
564void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb)
565{
566 iemThreadedTbFree(pVCpu->CTX_SUFF(pVM), pVCpu, pTb);
567}
568
569
570static PIEMTB iemThreadedTbLookup(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
571{
572 uint32_t const fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags | IEMTB_F_STATE_READY;
573 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, fFlags, GCPhysPc);
574 Log10(("TB lookup: idxHash=%#x fFlags=%#x GCPhysPc=%RGp\n", idxHash, fFlags, GCPhysPc));
575 PIEMTB pTb = g_TbCache.apHash[idxHash];
576 while (pTb)
577 {
578 if (pTb->GCPhysPc == GCPhysPc)
579 {
580 if (pTb->fFlags == fFlags)
581 {
582 if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
583 {
584#ifdef VBOX_WITH_STATISTICS
585 pVCpu->iem.s.cTbLookupHits++;
586#endif
587 return pTb;
588 }
589 Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
590 }
591 else
592 Log11(("TB miss: fFlags: %#x, wanted %#x\n", pTb->fFlags, fFlags));
593 }
594 else
595 Log11(("TB miss: GCPhysPc: %#x, wanted %#x\n", pTb->GCPhysPc, GCPhysPc));
596
597 pTb = pTb->pNext;
598 }
599 RT_NOREF(pVM);
600 pVCpu->iem.s.cTbLookupMisses++;
601 return pTb;
602}
603
604
605static void iemThreadedTbAdd(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
606{
607 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
608 pTb->pNext = g_TbCache.apHash[idxHash];
609 g_TbCache.apHash[idxHash] = pTb;
610 STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->iem.s.StatTbThreadedInstr, pTb->cInstructions);
611 STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->iem.s.StatTbThreadedCalls, pTb->Thrd.cCalls);
612 if (LogIs12Enabled())
613 {
614 Log12(("TB added: %p %RGp LB %#x fl=%#x idxHash=%#x cRanges=%u cInstr=%u cCalls=%u\n",
615 pTb, pTb->GCPhysPc, pTb->cbOpcodes, pTb->fFlags, idxHash, pTb->cRanges, pTb->cInstructions, pTb->Thrd.cCalls));
616 for (uint8_t idxRange = 0; idxRange < pTb->cRanges; idxRange++)
617 Log12((" range#%u: offPg=%#05x offOp=%#04x LB %#04x pg#%u=%RGp\n", idxRange, pTb->aRanges[idxRange].offPhysPage,
618 pTb->aRanges[idxRange].offOpcodes, pTb->aRanges[idxRange].cbOpcodes, pTb->aRanges[idxRange].idxPhysPage,
619 pTb->aRanges[idxRange].idxPhysPage == 0
620 ? pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK
621 : pTb->aGCPhysPages[pTb->aRanges[idxRange].idxPhysPage - 1]));
622 }
623 RT_NOREF(pVM);
624}
625
626
627/*
628 * Real code.
629 */
630
631#ifdef LOG_ENABLED
632/**
633 * Logs the current instruction.
634 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
635 * @param pszFunction The IEM function doing the execution.
636 */
637static void iemThreadedLogCurInstr(PVMCPUCC pVCpu, const char *pszFunction) RT_NOEXCEPT
638{
639# ifdef IN_RING3
640 if (LogIs2Enabled())
641 {
642 char szInstr[256];
643 uint32_t cbInstr = 0;
644 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
645 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
646 szInstr, sizeof(szInstr), &cbInstr);
647
648 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
649 Log2(("**** %s fExec=%x pTb=%p\n"
650 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
651 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
652 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
653 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
654 " %s\n"
655 , pszFunction, pVCpu->iem.s.fExec, pVCpu->iem.s.pCurTbR3,
656 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
657 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
658 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
659 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
660 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
661 szInstr));
662
663 if (LogIs3Enabled())
664 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
665 }
666 else
667# endif
668 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
669 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
670}
671#endif /* LOG_ENABLED */
672
673
674static VBOXSTRICTRC iemThreadedCompileLongJumped(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
675{
676 RT_NOREF(pVM, pVCpu);
677 return rcStrict;
678}
679
680
681/**
682 * Initializes the decoder state when compiling TBs.
683 *
684 * This presumes that fExec has already be initialized.
685 *
686 * This is very similar to iemInitDecoder() and iemReInitDecoder(), so may need
687 * to apply fixes to them as well.
688 *
689 * @param pVCpu The cross context virtual CPU structure of the calling
690 * thread.
691 * @param fReInit Clear for the first call for a TB, set for subsequent calls
692 * from inside the compile loop where we can skip a couple of
693 * things.
694 */
695DECL_FORCE_INLINE(void) iemThreadedCompileInitDecoder(PVMCPUCC pVCpu, bool const fReInit)
696{
697 /* ASSUMES: That iemInitExec was already called and that anyone changing
698 CPU state affecting the fExec bits since then will have updated fExec! */
699 AssertMsg((pVCpu->iem.s.fExec & ~IEM_F_USER_OPTS) == iemCalcExecFlags(pVCpu),
700 ("fExec=%#x iemCalcExecModeFlags=%#x\n", pVCpu->iem.s.fExec, iemCalcExecFlags(pVCpu)));
701
702 IEMMODE const enmMode = IEM_GET_CPU_MODE(pVCpu);
703
704 /* Decoder state: */
705 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
706 pVCpu->iem.s.enmEffAddrMode = enmMode;
707 if (enmMode != IEMMODE_64BIT)
708 {
709 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
710 pVCpu->iem.s.enmEffOpSize = enmMode;
711 }
712 else
713 {
714 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
715 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
716 }
717 pVCpu->iem.s.fPrefixes = 0;
718 pVCpu->iem.s.uRexReg = 0;
719 pVCpu->iem.s.uRexB = 0;
720 pVCpu->iem.s.uRexIndex = 0;
721 pVCpu->iem.s.idxPrefix = 0;
722 pVCpu->iem.s.uVex3rdReg = 0;
723 pVCpu->iem.s.uVexLength = 0;
724 pVCpu->iem.s.fEvexStuff = 0;
725 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
726 pVCpu->iem.s.offModRm = 0;
727 pVCpu->iem.s.iNextMapping = 0;
728
729 if (!fReInit)
730 {
731 pVCpu->iem.s.cActiveMappings = 0;
732 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
733 pVCpu->iem.s.fEndTb = false;
734 pVCpu->iem.s.fTbCheckOpcodes = false;
735 pVCpu->iem.s.fTbBranched = IEMBRANCHED_F_NO;
736 pVCpu->iem.s.fTbCrossedPage = false;
737 pVCpu->iem.s.cInstrTillIrqCheck = 32;
738 }
739 else
740 {
741 Assert(pVCpu->iem.s.cActiveMappings == 0);
742 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
743 Assert(pVCpu->iem.s.fEndTb == false);
744 Assert(pVCpu->iem.s.fTbCrossedPage == false);
745 }
746
747#ifdef DBGFTRACE_ENABLED
748 switch (IEM_GET_CPU_MODE(pVCpu))
749 {
750 case IEMMODE_64BIT:
751 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);
752 break;
753 case IEMMODE_32BIT:
754 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
755 break;
756 case IEMMODE_16BIT:
757 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
758 break;
759 }
760#endif
761}
762
763
764/**
765 * Initializes the opcode fetcher when starting the compilation.
766 *
767 * @param pVCpu The cross context virtual CPU structure of the calling
768 * thread.
769 */
770DECL_FORCE_INLINE(void) iemThreadedCompileInitOpcodeFetching(PVMCPUCC pVCpu)
771{
772 /* Almost everything is done by iemGetPcWithPhysAndCode() already. We just need to initialize the index into abOpcode. */
773#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
774 pVCpu->iem.s.offOpcode = 0;
775#else
776 RT_NOREF(pVCpu);
777#endif
778}
779
780
781/**
782 * Re-initializes the opcode fetcher between instructions while compiling.
783 *
784 * @param pVCpu The cross context virtual CPU structure of the calling
785 * thread.
786 */
787DECL_FORCE_INLINE(void) iemThreadedCompileReInitOpcodeFetching(PVMCPUCC pVCpu)
788{
789 if (pVCpu->iem.s.pbInstrBuf)
790 {
791 uint64_t off = pVCpu->cpum.GstCtx.rip;
792 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
793 off += pVCpu->cpum.GstCtx.cs.u64Base;
794 off -= pVCpu->iem.s.uInstrBufPc;
795 if (off < pVCpu->iem.s.cbInstrBufTotal)
796 {
797 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
798 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
799 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
800 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
801 else
802 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
803 }
804 else
805 {
806 pVCpu->iem.s.pbInstrBuf = NULL;
807 pVCpu->iem.s.offInstrNextByte = 0;
808 pVCpu->iem.s.offCurInstrStart = 0;
809 pVCpu->iem.s.cbInstrBuf = 0;
810 pVCpu->iem.s.cbInstrBufTotal = 0;
811 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
812 }
813 }
814 else
815 {
816 pVCpu->iem.s.offInstrNextByte = 0;
817 pVCpu->iem.s.offCurInstrStart = 0;
818 pVCpu->iem.s.cbInstrBuf = 0;
819 pVCpu->iem.s.cbInstrBufTotal = 0;
820#ifdef VBOX_STRICT
821 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
822#endif
823 }
824#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
825 pVCpu->iem.s.offOpcode = 0;
826#endif
827}
828
829
830DECLINLINE(void) iemThreadedCopyOpcodeBytesInline(PCVMCPUCC pVCpu, uint8_t *pbDst, uint8_t cbInstr)
831{
832 switch (cbInstr)
833 {
834 default: AssertMsgFailed(("%#x\n", cbInstr)); RT_FALL_THROUGH();
835 case 15: pbDst[14] = pVCpu->iem.s.abOpcode[14]; RT_FALL_THROUGH();
836 case 14: pbDst[13] = pVCpu->iem.s.abOpcode[13]; RT_FALL_THROUGH();
837 case 13: pbDst[12] = pVCpu->iem.s.abOpcode[12]; RT_FALL_THROUGH();
838 case 12: pbDst[11] = pVCpu->iem.s.abOpcode[11]; RT_FALL_THROUGH();
839 case 11: pbDst[10] = pVCpu->iem.s.abOpcode[10]; RT_FALL_THROUGH();
840 case 10: pbDst[9] = pVCpu->iem.s.abOpcode[9]; RT_FALL_THROUGH();
841 case 9: pbDst[8] = pVCpu->iem.s.abOpcode[8]; RT_FALL_THROUGH();
842 case 8: pbDst[7] = pVCpu->iem.s.abOpcode[7]; RT_FALL_THROUGH();
843 case 7: pbDst[6] = pVCpu->iem.s.abOpcode[6]; RT_FALL_THROUGH();
844 case 6: pbDst[5] = pVCpu->iem.s.abOpcode[5]; RT_FALL_THROUGH();
845 case 5: pbDst[4] = pVCpu->iem.s.abOpcode[4]; RT_FALL_THROUGH();
846 case 4: pbDst[3] = pVCpu->iem.s.abOpcode[3]; RT_FALL_THROUGH();
847 case 3: pbDst[2] = pVCpu->iem.s.abOpcode[2]; RT_FALL_THROUGH();
848 case 2: pbDst[1] = pVCpu->iem.s.abOpcode[1]; RT_FALL_THROUGH();
849 case 1: pbDst[0] = pVCpu->iem.s.abOpcode[0]; break;
850 }
851}
852
853
854/**
855 * Called by IEM_MC2_BEGIN_EMIT_CALLS() under one of these conditions:
856 *
857 * - CS LIM check required.
858 * - Must recheck opcode bytes.
859 * - Previous instruction branched.
860 * - TLB load detected, probably due to page crossing.
861 *
862 * @returns true if everything went well, false if we're out of space in the TB
863 * (e.g. opcode ranges).
864 * @param pVCpu The cross context virtual CPU structure of the calling
865 * thread.
866 * @param pTb The translation block being compiled.
867 */
868bool iemThreadedCompileBeginEmitCallsComplications(PVMCPUCC pVCpu, PIEMTB pTb)
869{
870 Assert((pVCpu->iem.s.GCPhysInstrBuf & GUEST_PAGE_OFFSET_MASK) == 0);
871#if 0
872 if (pVCpu->cpum.GstCtx.rip >= 0xc0000000 && !LogIsEnabled())
873 RTLogChangeFlags(NULL, 0, RTLOGFLAGS_DISABLED);
874#endif
875
876 /*
877 * Prepare call now, even before we know if can accept the instruction in this TB.
878 * This allows us amending parameters w/o making every case suffer.
879 */
880 uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu);
881 uint16_t const offOpcode = pTb->cbOpcodes;
882 uint8_t idxRange = pTb->cRanges - 1;
883
884 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls];
885 pCall->offOpcode = offOpcode;
886 pCall->idxRange = idxRange;
887 pCall->cbOpcode = cbInstr;
888 pCall->auParams[0] = cbInstr;
889 pCall->auParams[1] = idxRange;
890 pCall->auParams[2] = offOpcode - pTb->aRanges[idxRange].offOpcodes;
891
892/** @todo check if we require IEMTB_F_CS_LIM_CHECKS for any new page we've
893 * gotten onto. If we do, stop */
894
895 /*
896 * Case 1: We've branched (RIP changed).
897 *
898 * Sub-case 1a: Same page, no TLB load (fTbCrossedPage is false).
899 * Req: 1 extra range, no extra phys.
900 *
901 * Sub-case 1b: Different page but no page boundrary crossing, so TLB load
902 * necessary (fTbCrossedPage is true).
903 * Req: 1 extra range, probably 1 extra phys page entry.
904 *
905 * Sub-case 1c: Different page, so TLB load necessary (fTbCrossedPage is true),
906 * but in addition we cross into the following page and require
907 * another TLB load.
908 * Req: 2 extra ranges, probably 2 extra phys page entries.
909 *
910 * Sub-case 1d: Same page, so no initial TLB load necessary, but we cross into
911 * the following page (thus fTbCrossedPage is true).
912 * Req: 2 extra ranges, probably 1 extra phys page entry.
913 *
914 * Note! The setting fTbCrossedPage is done by the iemOpcodeFetchBytesJmp, but
915 * it may trigger "spuriously" from the CPU point of view because of
916 * physical page changes that'll invalid the physical TLB and trigger a
917 * call to the function. In theory this be a big deal, just a bit
918 * performance loss as we'll pick the LoadingTlb variants.
919 *
920 * Note! We do not currently optimize branching to the next instruction (sorry
921 * 32-bit PIC code). We could maybe do that in the branching code that
922 * sets (or not) fTbBranched.
923 */
924 /** @todo Optimize 'jmp .next_instr' and 'call .next_instr'. Seen the jmp
925 * variant in win 3.1 code and the call variant in 32-bit linux PIC
926 * code. This'll require filtering out far jmps and calls, as they
927 * load CS which should technically be considered indirect since the
928 * GDT/LDT entry's base address can be modified independently from
929 * the code. */
930 if (pVCpu->iem.s.fTbBranched != 0)
931 {
932 if ( !pVCpu->iem.s.fTbCrossedPage /* 1a */
933 || pVCpu->iem.s.offCurInstrStart >= 0 /* 1b */ )
934 {
935 /* 1a + 1b - instruction fully within the branched to page. */
936 Assert(pVCpu->iem.s.offCurInstrStart >= 0);
937 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr <= GUEST_PAGE_SIZE);
938
939 if (!(pVCpu->iem.s.fTbBranched & IEMBRANCHED_F_ZERO))
940 {
941 /* Check that we've got a free range. */
942 idxRange += 1;
943 if (idxRange < RT_ELEMENTS(pTb->aRanges))
944 { /* likely */ }
945 else
946 {
947 Log8(("%04x:%08RX64: out of ranges after branch\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
948 return false;
949 }
950 pCall->idxRange = idxRange;
951 pCall->auParams[1] = idxRange;
952 pCall->auParams[2] = 0;
953
954 /* Check that we've got a free page slot. */
955 AssertCompile(RT_ELEMENTS(pTb->aGCPhysPages) == 2);
956 RTGCPHYS const GCPhysNew = pVCpu->iem.s.GCPhysInstrBuf & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
957 if ((pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysNew)
958 pTb->aRanges[idxRange].idxPhysPage = 0;
959 else if ( pTb->aGCPhysPages[0] == NIL_RTGCPHYS
960 || pTb->aGCPhysPages[0] == GCPhysNew)
961 {
962 pTb->aGCPhysPages[0] = GCPhysNew;
963 pTb->aRanges[idxRange].idxPhysPage = 1;
964 }
965 else if ( pTb->aGCPhysPages[1] == NIL_RTGCPHYS
966 || pTb->aGCPhysPages[1] == GCPhysNew)
967 {
968 pTb->aGCPhysPages[1] = GCPhysNew;
969 pTb->aRanges[idxRange].idxPhysPage = 2;
970 }
971 else
972 {
973 Log8(("%04x:%08RX64: out of aGCPhysPages entires after branch\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
974 return false;
975 }
976
977 /* Finish setting up the new range. */
978 pTb->aRanges[idxRange].offPhysPage = pVCpu->iem.s.offCurInstrStart;
979 pTb->aRanges[idxRange].offOpcodes = offOpcode;
980 pTb->aRanges[idxRange].cbOpcodes = cbInstr;
981 pTb->aRanges[idxRange].u2Unused = 0;
982 pTb->cRanges++;
983 }
984 else
985 {
986 Log8(("%04x:%08RX64: zero byte jump\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
987 pTb->aRanges[idxRange].cbOpcodes += cbInstr;
988 }
989
990 /* Determin which function we need to load & check.
991 Note! For jumps to a new page, we'll set both fTbBranched and
992 fTbCrossedPage to avoid unnecessary TLB work for intra
993 page branching */
994 if ( (pVCpu->iem.s.fTbBranched & (IEMBRANCHED_F_INDIRECT | IEMBRANCHED_F_FAR)) /* Far is basically indirect. */
995 || pVCpu->iem.s.fTbCrossedPage)
996 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
997 ? kIemThreadedFunc_BltIn_CheckCsLimAndOpcodesLoadingTlb
998 : kIemThreadedFunc_BltIn_CheckOpcodesLoadingTlb;
999 else if (pVCpu->iem.s.fTbBranched & (IEMBRANCHED_F_CONDITIONAL | /* paranoia: */ IEMBRANCHED_F_DIRECT))
1000 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1001 ? kIemThreadedFunc_BltIn_CheckCsLimAndPcAndOpcodes
1002 : kIemThreadedFunc_BltIn_CheckPcAndOpcodes;
1003 else
1004 {
1005 Assert(pVCpu->iem.s.fTbBranched & IEMBRANCHED_F_RELATIVE);
1006 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1007 ? kIemThreadedFunc_BltIn_CheckCsLimAndOpcodes
1008 : kIemThreadedFunc_BltIn_CheckOpcodes;
1009 }
1010 }
1011 else
1012 {
1013 /* 1c + 1d - instruction crosses pages. */
1014 Assert(pVCpu->iem.s.offCurInstrStart < 0);
1015 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr > 0);
1016
1017 /* Lazy bird: Check that this isn't case 1c, since we've already
1018 load the first physical address. End the TB and
1019 make it a case 2b instead.
1020
1021 Hmm. Too much bother to detect, so just do the same
1022 with case 1d as well. */
1023#if 0 /** @todo get back to this later when we've got the actual branch code in
1024 * place. */
1025 uint8_t const cbStartPage = (uint8_t)-pVCpu->iem.s.offCurInstrStart;
1026
1027 /* Check that we've got two free ranges. */
1028 if (idxRange + 2 < RT_ELEMENTS(pTb->aRanges))
1029 { /* likely */ }
1030 else
1031 return false;
1032 idxRange += 1;
1033 pCall->idxRange = idxRange;
1034 pCall->auParams[1] = idxRange;
1035 pCall->auParams[2] = 0;
1036
1037 /* ... */
1038
1039#else
1040 Log8(("%04x:%08RX64: complicated post-branch condition, ending TB.\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1041 return false;
1042#endif
1043 }
1044 }
1045
1046 /*
1047 * Case 2: Page crossing.
1048 *
1049 * Sub-case 2a: The instruction starts on the first byte in the next page.
1050 *
1051 * Sub-case 2b: The instruction has opcode bytes in both the current and
1052 * following page.
1053 *
1054 * Both cases requires a new range table entry and probably a new physical
1055 * page entry. The difference is in which functions to emit and whether to
1056 * add bytes to the current range.
1057 */
1058 else if (pVCpu->iem.s.fTbCrossedPage)
1059 {
1060 /* Check that we've got a free range. */
1061 idxRange += 1;
1062 if (idxRange < RT_ELEMENTS(pTb->aRanges))
1063 { /* likely */ }
1064 else
1065 {
1066 Log8(("%04x:%08RX64: out of ranges while crossing page\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1067 return false;
1068 }
1069
1070 /* Check that we've got a free page slot. */
1071 AssertCompile(RT_ELEMENTS(pTb->aGCPhysPages) == 2);
1072 RTGCPHYS const GCPhysNew = pVCpu->iem.s.GCPhysInstrBuf & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1073 if ((pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysNew)
1074 pTb->aRanges[idxRange].idxPhysPage = 0;
1075 else if ( pTb->aGCPhysPages[0] == NIL_RTGCPHYS
1076 || pTb->aGCPhysPages[0] == GCPhysNew)
1077 {
1078 pTb->aGCPhysPages[0] = GCPhysNew;
1079 pTb->aRanges[idxRange].idxPhysPage = 1;
1080 }
1081 else if ( pTb->aGCPhysPages[1] == NIL_RTGCPHYS
1082 || pTb->aGCPhysPages[1] == GCPhysNew)
1083 {
1084 pTb->aGCPhysPages[1] = GCPhysNew;
1085 pTb->aRanges[idxRange].idxPhysPage = 2;
1086 }
1087 else
1088 {
1089 Log8(("%04x:%08RX64: out of aGCPhysPages entires while crossing page\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1090 return false;
1091 }
1092
1093 if (((pTb->aRanges[idxRange - 1].offPhysPage + pTb->aRanges[idxRange - 1].cbOpcodes) & GUEST_PAGE_OFFSET_MASK) == 0)
1094 {
1095 Assert(pVCpu->iem.s.offCurInstrStart == 0);
1096 pCall->idxRange = idxRange;
1097 pCall->auParams[1] = idxRange;
1098 pCall->auParams[2] = 0;
1099
1100 /* Finish setting up the new range. */
1101 pTb->aRanges[idxRange].offPhysPage = pVCpu->iem.s.offCurInstrStart;
1102 pTb->aRanges[idxRange].offOpcodes = offOpcode;
1103 pTb->aRanges[idxRange].cbOpcodes = cbInstr;
1104 pTb->aRanges[idxRange].u2Unused = 0;
1105 pTb->cRanges++;
1106
1107 /* Determin which function we need to load & check. */
1108 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1109 ? kIemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNewPageLoadingTlb
1110 : kIemThreadedFunc_BltIn_CheckOpcodesOnNewPageLoadingTlb;
1111 }
1112 else
1113 {
1114 Assert(pVCpu->iem.s.offCurInstrStart < 0);
1115 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr > 0);
1116 uint8_t const cbStartPage = (uint8_t)-pVCpu->iem.s.offCurInstrStart;
1117 pCall->auParams[0] |= (uint64_t)cbStartPage << 32;
1118
1119 /* We've good. Split the instruction over the old and new range table entries. */
1120 pTb->aRanges[idxRange - 1].cbOpcodes += cbStartPage;
1121
1122 pTb->aRanges[idxRange].offPhysPage = 0;
1123 pTb->aRanges[idxRange].offOpcodes = offOpcode + cbStartPage;
1124 pTb->aRanges[idxRange].cbOpcodes = cbInstr - cbStartPage;
1125 pTb->aRanges[idxRange].u2Unused = 0;
1126 pTb->cRanges++;
1127
1128 /* Determin which function we need to load & check. */
1129 if (pVCpu->iem.s.fTbCheckOpcodes)
1130 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1131 ? kIemThreadedFunc_BltIn_CheckCsLimAndOpcodesAcrossPageLoadingTlb
1132 : kIemThreadedFunc_BltIn_CheckOpcodesAcrossPageLoadingTlb;
1133 else
1134 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1135 ? kIemThreadedFunc_BltIn_CheckCsLimAndOpcodesOnNextPageLoadingTlb
1136 : kIemThreadedFunc_BltIn_CheckOpcodesOnNextPageLoadingTlb;
1137 }
1138 }
1139
1140 /*
1141 * Regular case: No new range required.
1142 */
1143 else
1144 {
1145 Assert(pVCpu->iem.s.fTbCheckOpcodes || (pTb->fFlags & IEMTB_F_CS_LIM_CHECKS));
1146 if (pVCpu->iem.s.fTbCheckOpcodes)
1147 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1148 ? kIemThreadedFunc_BltIn_CheckCsLimAndOpcodes
1149 : kIemThreadedFunc_BltIn_CheckOpcodes;
1150 else
1151 pCall->enmFunction = kIemThreadedFunc_BltIn_CheckCsLim;
1152
1153 iemThreadedCopyOpcodeBytesInline(pVCpu, &pTb->pabOpcodes[offOpcode], cbInstr);
1154 pTb->cbOpcodes = offOpcode + cbInstr;
1155 pTb->aRanges[idxRange].cbOpcodes += cbInstr;
1156 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1157 }
1158
1159 /*
1160 * Commit the call.
1161 */
1162 pTb->Thrd.cCalls++;
1163
1164 /*
1165 * Clear state.
1166 */
1167 pVCpu->iem.s.fTbBranched = IEMBRANCHED_F_NO;
1168 pVCpu->iem.s.fTbCrossedPage = false;
1169 pVCpu->iem.s.fTbCheckOpcodes = false;
1170
1171 /*
1172 * Copy opcode bytes.
1173 */
1174 iemThreadedCopyOpcodeBytesInline(pVCpu, &pTb->pabOpcodes[offOpcode], cbInstr);
1175 pTb->cbOpcodes = offOpcode + cbInstr;
1176 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1177
1178 return true;
1179}
1180
1181
1182/**
1183 * Emits an IRQ check call and checks for pending IRQs.
1184 *
1185 * @returns true if IRQs are pending and can be dispatched, false if not.
1186 * @param pVCpu The cross context virtual CPU structure of the calling
1187 * thread.
1188 * @param pTb The transation block.
1189 */
1190static bool iemThreadedCompileCheckIrq(PVMCPUCC pVCpu, PIEMTB pTb)
1191{
1192 pVCpu->iem.s.cInstrTillIrqCheck = 32;
1193
1194 /*
1195 * Emit the call first.
1196 */
1197 AssertReturn(pTb->Thrd.cCalls < pTb->Thrd.cAllocated, true);
1198 PIEMTHRDEDCALLENTRY pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls++];
1199 pCall->enmFunction = kIemThreadedFunc_BltIn_CheckIrq;
1200 pCall->uUnused0 = 0;
1201 pCall->offOpcode = 0;
1202 pCall->cbOpcode = 0;
1203 pCall->idxRange = 0;
1204 pCall->auParams[0] = 0;
1205 pCall->auParams[1] = 0;
1206 pCall->auParams[2] = 0;
1207
1208
1209 /*
1210 * Check for IRQs.
1211 */
1212 uint64_t fCpu = pVCpu->fLocalForcedActions;
1213 fCpu &= VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI;
1214 if (RT_LIKELY( !fCpu
1215 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
1216 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF)))
1217 return false;
1218 return true;
1219}
1220
1221
1222/**
1223 * Compiles a new TB and executes it.
1224 *
1225 * We combine compilation and execution here as it makes it simpler code flow
1226 * in the main loop and it allows interpreting while compiling if we want to
1227 * explore that option.
1228 *
1229 * @returns Strict VBox status code.
1230 * @param pVM The cross context virtual machine structure.
1231 * @param pVCpu The cross context virtual CPU structure of the calling
1232 * thread.
1233 * @param GCPhysPc The physical address corresponding to the current
1234 * RIP+CS.BASE.
1235 * @param fExtraFlags Extra translation block flags: IEMTB_F_TYPE_THREADED and
1236 * maybe IEMTB_F_RIP_CHECKS.
1237 */
1238static VBOXSTRICTRC iemThreadedCompile(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
1239{
1240 /*
1241 * Allocate a new translation block.
1242 */
1243 PIEMTB pTb = iemThreadedTbAlloc(pVM, pVCpu, GCPhysPc, fExtraFlags | IEMTB_F_STATE_COMPILING);
1244 AssertReturn(pTb, VERR_IEM_TB_ALLOC_FAILED);
1245
1246 /* Set the current TB so iemThreadedCompileLongJumped and the CIMPL
1247 functions may get at it. */
1248 pVCpu->iem.s.pCurTbR3 = pTb;
1249
1250 /*
1251 * Now for the recomplication. (This mimicks IEMExecLots in many ways.)
1252 */
1253 iemThreadedCompileInitDecoder(pVCpu, false /*fReInit*/);
1254 iemThreadedCompileInitOpcodeFetching(pVCpu);
1255 VBOXSTRICTRC rcStrict;
1256 for (;;)
1257 {
1258 /* Process the next instruction. */
1259#ifdef LOG_ENABLED
1260 iemThreadedLogCurInstr(pVCpu, "CC");
1261 uint16_t const uCsLog = pVCpu->cpum.GstCtx.cs.Sel;
1262 uint64_t const uRipLog = pVCpu->cpum.GstCtx.rip;
1263#endif
1264 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
1265 uint16_t const cCallsPrev = pTb->Thrd.cCalls;
1266
1267 rcStrict = FNIEMOP_CALL(g_apfnIemThreadedRecompilerOneByteMap[b]);
1268 if ( rcStrict == VINF_SUCCESS
1269 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS
1270 && !pVCpu->iem.s.fEndTb)
1271 {
1272 Assert(pTb->Thrd.cCalls > cCallsPrev);
1273 Assert(cCallsPrev - pTb->Thrd.cCalls < 5);
1274
1275 pVCpu->iem.s.cInstructions++;
1276 }
1277 else
1278 {
1279 Log8(("%04x:%08RX64: End TB - %u instr, %u calls, rc=%d\n",
1280 uCsLog, uRipLog, pTb->cInstructions, pTb->Thrd.cCalls, VBOXSTRICTRC_VAL(rcStrict)));
1281 if (rcStrict == VINF_IEM_RECOMPILE_END_TB)
1282 rcStrict = VINF_SUCCESS;
1283
1284 if (pTb->Thrd.cCalls > 0)
1285 {
1286 if (cCallsPrev != pTb->Thrd.cCalls)
1287 pVCpu->iem.s.cInstructions++;
1288 break;
1289 }
1290
1291 pVCpu->iem.s.pCurTbR3 = NULL;
1292 iemThreadedTbFree(pVM, pVCpu, pTb);
1293 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1294 }
1295
1296 /* Check for IRQs? */
1297 if (pVCpu->iem.s.cInstrTillIrqCheck-- > 0)
1298 { /* likely */ }
1299 else if (iemThreadedCompileCheckIrq(pVCpu, pTb))
1300 break;
1301
1302 /* Still space in the TB? */
1303 if ( pTb->Thrd.cCalls + 5 < pTb->Thrd.cAllocated
1304 && pTb->cbOpcodes + 16 <= pTb->cbOpcodesAllocated)
1305 iemThreadedCompileInitDecoder(pVCpu, true /*fReInit*/);
1306 else
1307 {
1308 Log8(("%04x:%08RX64: End TB - %u instr, %u calls, %u opcode bytes - full\n",
1309 uCsLog, uRipLog, pTb->cInstructions, pTb->Thrd.cCalls, pTb->cbOpcodes));
1310 break;
1311 }
1312 iemThreadedCompileReInitOpcodeFetching(pVCpu);
1313 }
1314
1315 /*
1316 * Complete the TB and link it.
1317 */
1318 pTb->fFlags = (pTb->fFlags & ~IEMTB_F_STATE_MASK) | IEMTB_F_STATE_READY;
1319 iemThreadedTbAdd(pVM, pVCpu, pTb);
1320
1321#ifdef IEM_COMPILE_ONLY_MODE
1322 /*
1323 * Execute the translation block.
1324 */
1325#endif
1326
1327 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1328}
1329
1330
1331/**
1332 * Executes a translation block.
1333 *
1334 * @returns Strict VBox status code.
1335 * @param pVCpu The cross context virtual CPU structure of the calling
1336 * thread.
1337 * @param pTb The translation block to execute.
1338 */
1339static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb) IEM_NOEXCEPT_MAY_LONGJMP
1340{
1341 /* Check the opcodes in the first page before starting execution. */
1342 Assert(!(pVCpu->iem.s.GCPhysInstrBuf & (RTGCPHYS)GUEST_PAGE_OFFSET_MASK));
1343 Assert(pTb->aRanges[0].cbOpcodes <= pVCpu->iem.s.cbInstrBufTotal - pVCpu->iem.s.offInstrNextByte);
1344 if (memcmp(pTb->pabOpcodes, &pVCpu->iem.s.pbInstrBuf[pTb->aRanges[0].offPhysPage], pTb->aRanges[0].cbOpcodes) == 0)
1345 { /* likely */ }
1346 else
1347 {
1348 Log7(("TB obsolete: %p GCPhys=%RGp\n", pTb, pTb->GCPhysPc));
1349 iemThreadedTbFree(pVCpu->pVMR3, pVCpu, pTb);
1350 return VINF_SUCCESS;
1351 }
1352
1353 /* Set the current TB so CIMPL function may get at it. */
1354 pVCpu->iem.s.pCurTbR3 = pTb;
1355 pVCpu->iem.s.cTbExec++;
1356
1357 /* The execution loop. */
1358 PCIEMTHRDEDCALLENTRY pCallEntry = pTb->Thrd.paCalls;
1359 uint32_t cCallsLeft = pTb->Thrd.cCalls;
1360 while (cCallsLeft-- > 0)
1361 {
1362#ifdef LOG_ENABLED
1363 iemThreadedLogCurInstr(pVCpu, "EX");
1364 Log9(("%04x:%08RX64: #%d - %d %s\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
1365 pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->enmFunction, g_apszIemThreadedFunctions[pCallEntry->enmFunction]));
1366#endif
1367 VBOXSTRICTRC const rcStrict = g_apfnIemThreadedFunctions[pCallEntry->enmFunction](pVCpu,
1368 pCallEntry->auParams[0],
1369 pCallEntry->auParams[1],
1370 pCallEntry->auParams[2]);
1371
1372 if (RT_LIKELY( rcStrict == VINF_SUCCESS
1373 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
1374 pCallEntry++;
1375 else
1376 {
1377 pVCpu->iem.s.pCurTbR3 = NULL;
1378
1379 /* Some status codes are just to get us out of this loop and
1380 continue in a different translation block. */
1381 if (rcStrict == VINF_IEM_REEXEC_MODE_CHANGED)
1382 return iemExecStatusCodeFiddling(pVCpu, VINF_SUCCESS);
1383 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1384 }
1385 }
1386
1387 pVCpu->iem.s.pCurTbR3 = NULL;
1388 return VINF_SUCCESS;
1389}
1390
1391
1392/**
1393 * This is called when the PC doesn't match the current pbInstrBuf.
1394 *
1395 * Upon return, we're ready for opcode fetching. But please note that
1396 * pbInstrBuf can be NULL iff the memory doesn't have readable backing (i.e.
1397 * MMIO or unassigned).
1398 */
1399static RTGCPHYS iemGetPcWithPhysAndCodeMissed(PVMCPUCC pVCpu)
1400{
1401 pVCpu->iem.s.pbInstrBuf = NULL;
1402 pVCpu->iem.s.offCurInstrStart = 0;
1403 pVCpu->iem.s.offInstrNextByte = 0;
1404 iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
1405 return pVCpu->iem.s.GCPhysInstrBuf + pVCpu->iem.s.offCurInstrStart;
1406}
1407
1408
1409/** @todo need private inline decl for throw/nothrow matching IEM_WITH_SETJMP? */
1410DECL_FORCE_INLINE_THROW(RTGCPHYS) iemGetPcWithPhysAndCode(PVMCPUCC pVCpu)
1411{
1412 /*
1413 * Set uCurTbStartPc to RIP and calc the effective PC.
1414 */
1415 uint64_t uPc = pVCpu->cpum.GstCtx.rip;
1416 pVCpu->iem.s.uCurTbStartPc = uPc;
1417 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
1418 uPc += pVCpu->cpum.GstCtx.cs.u64Base;
1419
1420 /*
1421 * Advance within the current buffer (PAGE) when possible.
1422 */
1423 if (pVCpu->iem.s.pbInstrBuf)
1424 {
1425 uint64_t off = uPc - pVCpu->iem.s.uInstrBufPc;
1426 if (off < pVCpu->iem.s.cbInstrBufTotal)
1427 {
1428 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1429 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1430 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1431 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1432 else
1433 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1434
1435 return pVCpu->iem.s.GCPhysInstrBuf + off;
1436 }
1437 }
1438 return iemGetPcWithPhysAndCodeMissed(pVCpu);
1439}
1440
1441
1442/**
1443 * Determines the extra IEMTB_F_XXX flags.
1444 *
1445 * @returns IEMTB_F_TYPE_THREADED and maybe IEMTB_F_RIP_CHECKS.
1446 * @param pVCpu The cross context virtual CPU structure of the calling
1447 * thread.
1448 */
1449DECL_FORCE_INLINE(uint32_t) iemGetTbFlagsForCurrentPc(PVMCPUCC pVCpu)
1450{
1451 /*
1452 * Return IEMTB_F_RIP_CHECKS if the current PC is invalid or if it is
1453 * likely to go invalid before the end of the translation block.
1454 */
1455 if (IEM_IS_64BIT_CODE(pVCpu))
1456 return IEMTB_F_TYPE_THREADED;
1457
1458 if (RT_LIKELY( pVCpu->cpum.GstCtx.eip < pVCpu->cpum.GstCtx.cs.u32Limit
1459 && pVCpu->cpum.GstCtx.eip - pVCpu->cpum.GstCtx.cs.u32Limit >= X86_PAGE_SIZE))
1460 return IEMTB_F_TYPE_THREADED;
1461
1462 return IEMTB_F_TYPE_THREADED | IEMTB_F_CS_LIM_CHECKS;
1463}
1464
1465
1466VMMDECL(VBOXSTRICTRC) IEMExecRecompilerThreaded(PVMCC pVM, PVMCPUCC pVCpu)
1467{
1468 /*
1469 * See if there is an interrupt pending in TRPM, inject it if we can.
1470 */
1471 if (!TRPMHasTrap(pVCpu))
1472 { /* likely */ }
1473 else
1474 {
1475 VBOXSTRICTRC rcStrict = iemExecInjectPendingTrap(pVCpu);
1476 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1477 { /*likely */ }
1478 else
1479 return rcStrict;
1480 }
1481
1482 /*
1483 * Init the execution environment.
1484 */
1485 iemInitExec(pVCpu, 0 /*fExecOpts*/);
1486
1487 /*
1488 * Run-loop.
1489 *
1490 * If we're using setjmp/longjmp we combine all the catching here to avoid
1491 * having to call setjmp for each block we're executing.
1492 */
1493 for (;;)
1494 {
1495 PIEMTB pTb = NULL;
1496 VBOXSTRICTRC rcStrict;
1497 IEM_TRY_SETJMP(pVCpu, rcStrict)
1498 {
1499 uint32_t const cPollRate = 511; /* EM.cpp passes 4095 to IEMExecLots, so an eigth of that seems reasonable for now. */
1500 for (uint32_t iIterations = 0; ; iIterations++)
1501 {
1502 /* Translate PC to physical address, we'll need this for both lookup and compilation. */
1503 RTGCPHYS const GCPhysPc = iemGetPcWithPhysAndCode(pVCpu);
1504 uint32_t const fExtraFlags = iemGetTbFlagsForCurrentPc(pVCpu);
1505
1506 pTb = iemThreadedTbLookup(pVM, pVCpu, GCPhysPc, fExtraFlags);
1507 if (pTb)
1508 rcStrict = iemThreadedTbExec(pVCpu, pTb);
1509 else
1510 rcStrict = iemThreadedCompile(pVM, pVCpu, GCPhysPc, fExtraFlags);
1511 if (rcStrict == VINF_SUCCESS)
1512 {
1513 Assert(pVCpu->iem.s.cActiveMappings == 0);
1514
1515 uint64_t fCpu = pVCpu->fLocalForcedActions;
1516 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
1517 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
1518 | VMCPU_FF_TLB_FLUSH
1519 | VMCPU_FF_UNHALT );
1520 if (RT_LIKELY( ( !fCpu
1521 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
1522 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
1523 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
1524 {
1525 if (RT_LIKELY( (iIterations & cPollRate) != 0
1526 || !TMTimerPollBool(pVM, pVCpu)))
1527 {
1528
1529 }
1530 else
1531 return VINF_SUCCESS;
1532 }
1533 else
1534 return VINF_SUCCESS;
1535 }
1536 else
1537 return rcStrict;
1538 }
1539 }
1540 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
1541 {
1542 pVCpu->iem.s.cLongJumps++;
1543 if (pVCpu->iem.s.cActiveMappings > 0)
1544 iemMemRollback(pVCpu);
1545
1546 /* If pTb isn't NULL we're in iemThreadedTbExec. */
1547 if (!pTb)
1548 {
1549 /* If pCurTbR3 is NULL, we're in iemGetPcWithPhysAndCode.*/
1550 pTb = pVCpu->iem.s.pCurTbR3;
1551 if (pTb)
1552 {
1553 /* If the pCurTbR3 block is in compiling state, we're in iemThreadedCompile,
1554 otherwise it's iemThreadedTbExec inside iemThreadedCompile (compile option). */
1555 if ((pTb->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_COMPILING)
1556 return iemThreadedCompileLongJumped(pVM, pVCpu, rcStrict);
1557 }
1558 }
1559 return rcStrict;
1560 }
1561 IEM_CATCH_LONGJMP_END(pVCpu);
1562 }
1563}
1564
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette