VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllThrdRecompiler.cpp@ 100759

Last change on this file since 100759 was 100752, checked in by vboxsync, 17 months ago

VMM/IEM: Correctly declare the threaded functions (Visual C++ freaked out while throwing stuff again). Windows build fixes. bugref:10369
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 60.0 KB
Line 
1/* $Id: IEMAllThrdRecompiler.cpp 100752 2023-07-31 13:24:18Z vboxsync $ */
2/** @file
3 * IEM - Instruction Decoding and Threaded Recompilation.
4 *
5 * Logging group IEM_RE_THREADED assignments:
6 * - Level 1 (Log) : Errors, exceptions, interrupts and such major events. [same as IEM]
7 * - Flow (LogFlow) :
8 * - Level 2 (Log2) : Basic instruction execution state info. [same as IEM]
9 * - Level 3 (Log3) : More detailed execution state info. [same as IEM]
10 * - Level 4 (Log4) : Decoding mnemonics w/ EIP. [same as IEM]
11 * - Level 5 (Log5) : Decoding details. [same as IEM]
12 * - Level 6 (Log6) :
13 * - Level 7 (Log7) : TB obsoletion.
14 * - Level 8 (Log8) : TB compilation.
15 * - Level 9 (Log9) : TB exec.
16 * - Level 10 (Log10): TB block lookup.
17 * - Level 11 (Log11): TB block lookup details.
18 * - Level 12 (Log12): TB insertion.
19 */
20
21/*
22 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
23 *
24 * This file is part of VirtualBox base platform packages, as
25 * available from https://www.virtualbox.org.
26 *
27 * This program is free software; you can redistribute it and/or
28 * modify it under the terms of the GNU General Public License
29 * as published by the Free Software Foundation, in version 3 of the
30 * License.
31 *
32 * This program is distributed in the hope that it will be useful, but
33 * WITHOUT ANY WARRANTY; without even the implied warranty of
34 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
35 * General Public License for more details.
36 *
37 * You should have received a copy of the GNU General Public License
38 * along with this program; if not, see <https://www.gnu.org/licenses>.
39 *
40 * SPDX-License-Identifier: GPL-3.0-only
41 */
42
43
44/*********************************************************************************************************************************
45* Header Files *
46*********************************************************************************************************************************/
47#ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */
48# define LOG_GROUP LOG_GROUP_IEM_RE_THREADED
49#endif
50#define IEM_WITH_CODE_TLB_AND_OPCODE_BUF /* A bit hackish, but its all in IEMInline.h. */
51#define VMCPU_INCL_CPUM_GST_CTX
52#include <VBox/vmm/iem.h>
53#include <VBox/vmm/cpum.h>
54#include <VBox/vmm/apic.h>
55#include <VBox/vmm/pdm.h>
56#include <VBox/vmm/pgm.h>
57#include <VBox/vmm/iom.h>
58#include <VBox/vmm/em.h>
59#include <VBox/vmm/hm.h>
60#include <VBox/vmm/nem.h>
61#include <VBox/vmm/gim.h>
62#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
63# include <VBox/vmm/em.h>
64# include <VBox/vmm/hm_svm.h>
65#endif
66#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
67# include <VBox/vmm/hmvmxinline.h>
68#endif
69#include <VBox/vmm/tm.h>
70#include <VBox/vmm/dbgf.h>
71#include <VBox/vmm/dbgftrace.h>
72#ifndef TST_IEM_CHECK_MC
73# include "IEMInternal.h"
74#endif
75#include <VBox/vmm/vmcc.h>
76#include <VBox/log.h>
77#include <VBox/err.h>
78#include <VBox/param.h>
79#include <VBox/dis.h>
80#include <VBox/disopcode-x86-amd64.h>
81#include <iprt/asm-math.h>
82#include <iprt/assert.h>
83#include <iprt/mem.h>
84#include <iprt/string.h>
85#include <iprt/x86.h>
86
87#ifndef TST_IEM_CHECK_MC
88# include "IEMInline.h"
89# include "IEMOpHlp.h"
90# include "IEMMc.h"
91#endif
92
93#include "IEMThreadedFunctions.h"
94
95
96/*
97 * Narrow down configs here to avoid wasting time on unused configs here.
98 */
99
100#ifndef IEM_WITH_CODE_TLB
101# error The code TLB must be enabled for the recompiler.
102#endif
103
104#ifndef IEM_WITH_DATA_TLB
105# error The data TLB must be enabled for the recompiler.
106#endif
107
108#ifndef IEM_WITH_SETJMP
109# error The setjmp approach must be enabled for the recompiler.
110#endif
111
112
113/*********************************************************************************************************************************
114* Internal Functions *
115*********************************************************************************************************************************/
116static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb);
117
118
119/**
120 * Calculates the effective address of a ModR/M memory operand, extended version
121 * for use in the recompilers.
122 *
123 * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
124 *
125 * May longjmp on internal error.
126 *
127 * @return The effective address.
128 * @param pVCpu The cross context virtual CPU structure of the calling thread.
129 * @param bRm The ModRM byte.
130 * @param cbImmAndRspOffset - First byte: The size of any immediate
131 * following the effective address opcode bytes
132 * (only for RIP relative addressing).
133 * - Second byte: RSP displacement (for POP [ESP]).
134 * @param puInfo Extra info: 32-bit displacement (bits 31:0) and
135 * SIB byte (bits 39:32).
136 *
137 * @note This must be defined in a source file with matching
138 * IEM_WITH_CODE_TLB_AND_OPCODE_BUF define till the define is made default
139 * or implemented differently...
140 */
141RTGCPTR iemOpHlpCalcRmEffAddrJmpEx(PVMCPUCC pVCpu, uint8_t bRm, uint32_t cbImmAndRspOffset, uint64_t *puInfo) IEM_NOEXCEPT_MAY_LONGJMP
142{
143 Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
144# define SET_SS_DEF() \
145 do \
146 { \
147 if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
148 pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
149 } while (0)
150
151 if (!IEM_IS_64BIT_CODE(pVCpu))
152 {
153/** @todo Check the effective address size crap! */
154 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
155 {
156 uint16_t u16EffAddr;
157
158 /* Handle the disp16 form with no registers first. */
159 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
160 {
161 IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
162 *puInfo = u16EffAddr;
163 }
164 else
165 {
166 /* Get the displacment. */
167 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
168 {
169 case 0: u16EffAddr = 0; break;
170 case 1: IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
171 case 2: IEM_OPCODE_GET_NEXT_U16(&u16EffAddr); break;
172 default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */
173 }
174 *puInfo = u16EffAddr;
175
176 /* Add the base and index registers to the disp. */
177 switch (bRm & X86_MODRM_RM_MASK)
178 {
179 case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
180 case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
181 case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
182 case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
183 case 4: u16EffAddr += pVCpu->cpum.GstCtx.si; break;
184 case 5: u16EffAddr += pVCpu->cpum.GstCtx.di; break;
185 case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp; SET_SS_DEF(); break;
186 case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx; break;
187 }
188 }
189
190 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16 uInfo=%#RX64\n", u16EffAddr, *puInfo));
191 return u16EffAddr;
192 }
193
194 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
195 uint32_t u32EffAddr;
196 uint64_t uInfo;
197
198 /* Handle the disp32 form with no registers first. */
199 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
200 {
201 IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
202 uInfo = u32EffAddr;
203 }
204 else
205 {
206 /* Get the register (or SIB) value. */
207 uInfo = 0;
208 switch ((bRm & X86_MODRM_RM_MASK))
209 {
210 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
211 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
212 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
213 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
214 case 4: /* SIB */
215 {
216 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
217 uInfo = (uint64_t)bSib << 32;
218
219 /* Get the index and scale it. */
220 switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
221 {
222 case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
223 case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
224 case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
225 case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
226 case 4: u32EffAddr = 0; /*none */ break;
227 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
228 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
229 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
230 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
231 }
232 u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
233
234 /* add base */
235 switch (bSib & X86_SIB_BASE_MASK)
236 {
237 case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
238 case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
239 case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
240 case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
241 case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
242 case 5:
243 if ((bRm & X86_MODRM_MOD_MASK) != 0)
244 {
245 u32EffAddr += pVCpu->cpum.GstCtx.ebp;
246 SET_SS_DEF();
247 }
248 else
249 {
250 uint32_t u32Disp;
251 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
252 u32EffAddr += u32Disp;
253 uInfo |= u32Disp;
254 }
255 break;
256 case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
257 case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
258 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
259 }
260 break;
261 }
262 case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
263 case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
264 case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
265 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
266 }
267
268 /* Get and add the displacement. */
269 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
270 {
271 case 0:
272 break;
273 case 1:
274 {
275 int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
276 u32EffAddr += i8Disp;
277 uInfo |= (uint32_t)(int32_t)i8Disp;
278 break;
279 }
280 case 2:
281 {
282 uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
283 u32EffAddr += u32Disp;
284 uInfo |= u32Disp;
285 break;
286 }
287 default:
288 AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */
289 }
290 }
291
292 *puInfo = uInfo;
293 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32 uInfo=%#RX64\n", u32EffAddr, uInfo));
294 return u32EffAddr;
295 }
296
297 uint64_t u64EffAddr;
298 uint64_t uInfo;
299
300 /* Handle the rip+disp32 form with no registers first. */
301 if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
302 {
303 IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
304 uInfo = (uint32_t)u64EffAddr;
305 u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + (cbImmAndRspOffset & UINT32_C(0xff));
306 }
307 else
308 {
309 /* Get the register (or SIB) value. */
310 uInfo = 0;
311 switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
312 {
313 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
314 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
315 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
316 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
317 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
318 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
319 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
320 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
321 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
322 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
323 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
324 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
325 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
326 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
327 /* SIB */
328 case 4:
329 case 12:
330 {
331 uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
332 uInfo = (uint64_t)bSib << 32;
333
334 /* Get the index and scale it. */
335 switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
336 {
337 case 0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
338 case 1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
339 case 2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
340 case 3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
341 case 4: u64EffAddr = 0; /*none */ break;
342 case 5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
343 case 6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
344 case 7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
345 case 8: u64EffAddr = pVCpu->cpum.GstCtx.r8; break;
346 case 9: u64EffAddr = pVCpu->cpum.GstCtx.r9; break;
347 case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
348 case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
349 case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
350 case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
351 case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
352 case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
353 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
354 }
355 u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
356
357 /* add base */
358 switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
359 {
360 case 0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
361 case 1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
362 case 2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
363 case 3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
364 case 4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + (cbImmAndRspOffset >> 8); SET_SS_DEF(); break;
365 case 6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
366 case 7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
367 case 8: u64EffAddr += pVCpu->cpum.GstCtx.r8; break;
368 case 9: u64EffAddr += pVCpu->cpum.GstCtx.r9; break;
369 case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
370 case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
371 case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
372 case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
373 case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
374 /* complicated encodings */
375 case 5:
376 case 13:
377 if ((bRm & X86_MODRM_MOD_MASK) != 0)
378 {
379 if (!pVCpu->iem.s.uRexB)
380 {
381 u64EffAddr += pVCpu->cpum.GstCtx.rbp;
382 SET_SS_DEF();
383 }
384 else
385 u64EffAddr += pVCpu->cpum.GstCtx.r13;
386 }
387 else
388 {
389 uint32_t u32Disp;
390 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
391 u64EffAddr += (int32_t)u32Disp;
392 uInfo |= u32Disp;
393 }
394 break;
395 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
396 }
397 break;
398 }
399 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
400 }
401
402 /* Get and add the displacement. */
403 switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
404 {
405 case 0:
406 break;
407 case 1:
408 {
409 int8_t i8Disp;
410 IEM_OPCODE_GET_NEXT_S8(&i8Disp);
411 u64EffAddr += i8Disp;
412 uInfo |= (uint32_t)(int32_t)i8Disp;
413 break;
414 }
415 case 2:
416 {
417 uint32_t u32Disp;
418 IEM_OPCODE_GET_NEXT_U32(&u32Disp);
419 u64EffAddr += (int32_t)u32Disp;
420 uInfo |= u32Disp;
421 break;
422 }
423 IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
424 }
425
426 }
427
428 *puInfo = uInfo;
429 if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
430 {
431 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr, uInfo));
432 return u64EffAddr;
433 }
434 Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
435 Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv uInfo=%#RX64\n", u64EffAddr & UINT32_MAX, uInfo));
436 return u64EffAddr & UINT32_MAX;
437}
438
439
440/*
441 * Translation block management.
442 */
443
444typedef struct IEMTBCACHE
445{
446 uint32_t cHash;
447 uint32_t uHashMask;
448 PIEMTB apHash[_64K];
449} IEMTBCACHE;
450
451static IEMTBCACHE g_TbCache = { _64K, 0xffff, }; /**< Quick and dirty. */
452
453#define IEMTBCACHE_HASH(a_paCache, a_fTbFlags, a_GCPhysPc) \
454 ( ((uint32_t)(a_GCPhysPc) ^ (a_fTbFlags)) & (a_paCache)->uHashMask)
455
456
457/**
458 * Allocate a translation block for threadeded recompilation.
459 *
460 * @returns Pointer to the translation block on success, NULL on failure.
461 * @param pVM The cross context virtual machine structure.
462 * @param pVCpu The cross context virtual CPU structure of the calling
463 * thread.
464 * @param GCPhysPc The physical address corresponding to RIP + CS.BASE.
465 * @param fExtraFlags Extra flags (IEMTB_F_XXX).
466 */
467static PIEMTB iemThreadedTbAlloc(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags)
468{
469 /*
470 * Just using the heap for now. Will make this more efficient and
471 * complicated later, don't worry. :-)
472 */
473 PIEMTB pTb = (PIEMTB)RTMemAlloc(sizeof(IEMTB));
474 if (pTb)
475 {
476 unsigned const cCalls = 128;
477 pTb->Thrd.paCalls = (PIEMTHRDEDCALLENTRY)RTMemAlloc(sizeof(IEMTHRDEDCALLENTRY) * cCalls);
478 if (pTb->Thrd.paCalls)
479 {
480 pTb->pabOpcodes = (uint8_t *)RTMemAlloc(cCalls * 16); /* This will be reallocated later. */
481 if (pTb->pabOpcodes)
482 {
483 pTb->Thrd.cAllocated = cCalls;
484 pTb->cbOpcodesAllocated = cCalls * 16;
485 pTb->Thrd.cCalls = 0;
486 pTb->cbOpcodes = 0;
487 pTb->pNext = NULL;
488 RTListInit(&pTb->LocalList);
489 pTb->GCPhysPc = GCPhysPc;
490 pTb->x86.fAttr = (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u;
491 pTb->fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags;
492 pTb->cInstructions = 0;
493
494 /* Init the first opcode range. */
495 pTb->cRanges = 1;
496 pTb->aRanges[0].cbOpcodes = 0;
497 pTb->aRanges[0].offOpcodes = 0;
498 pTb->aRanges[0].offPhysPage = GCPhysPc & GUEST_PAGE_OFFSET_MASK;
499 pTb->aRanges[0].u2Unused = 0;
500 pTb->aRanges[0].idxPhysPage = 0;
501 pTb->aGCPhysPages[0] = NIL_RTGCPHYS;
502 pTb->aGCPhysPages[1] = NIL_RTGCPHYS;
503
504 pVCpu->iem.s.cTbAllocs++;
505 return pTb;
506 }
507 RTMemFree(pTb->Thrd.paCalls);
508 }
509 RTMemFree(pTb);
510 }
511 RT_NOREF(pVM);
512 return NULL;
513}
514
515
516/**
517 * Frees pTb.
518 *
519 * @param pVM The cross context virtual machine structure.
520 * @param pVCpu The cross context virtual CPU structure of the calling
521 * thread.
522 * @param pTb The translation block to free..
523 */
524static void iemThreadedTbFree(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
525{
526 RT_NOREF(pVM);
527 AssertPtr(pTb);
528
529 AssertCompile(IEMTB_F_STATE_OBSOLETE == IEMTB_F_STATE_MASK);
530 pTb->fFlags |= IEMTB_F_STATE_OBSOLETE; /* works, both bits set */
531
532 /* Unlink it from the hash table: */
533 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
534 PIEMTB pTbCur = g_TbCache.apHash[idxHash];
535 if (pTbCur == pTb)
536 g_TbCache.apHash[idxHash] = pTb->pNext;
537 else
538 while (pTbCur)
539 {
540 PIEMTB const pNextTb = pTbCur->pNext;
541 if (pNextTb == pTb)
542 {
543 pTbCur->pNext = pTb->pNext;
544 break;
545 }
546 pTbCur = pNextTb;
547 }
548
549 /* Free it. */
550 RTMemFree(pTb->Thrd.paCalls);
551 pTb->Thrd.paCalls = NULL;
552
553 RTMemFree(pTb->pabOpcodes);
554 pTb->pabOpcodes = NULL;
555
556 RTMemFree(pTb);
557 pVCpu->iem.s.cTbFrees++;
558}
559
560
561/**
562 * Called by opcode verifier functions when they detect a problem.
563 */
564void iemThreadedTbObsolete(PVMCPUCC pVCpu, PIEMTB pTb)
565{
566 iemThreadedTbFree(pVCpu->CTX_SUFF(pVM), pVCpu, pTb);
567}
568
569
570static PIEMTB iemThreadedTbLookup(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
571{
572 uint32_t const fFlags = (pVCpu->iem.s.fExec & IEMTB_F_IEM_F_MASK) | fExtraFlags | IEMTB_F_STATE_READY;
573 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, fFlags, GCPhysPc);
574 Log10(("TB lookup: idxHash=%#x fFlags=%#x GCPhysPc=%RGp\n", idxHash, fFlags, GCPhysPc));
575 PIEMTB pTb = g_TbCache.apHash[idxHash];
576 while (pTb)
577 {
578 if (pTb->GCPhysPc == GCPhysPc)
579 {
580 if (pTb->fFlags == fFlags)
581 {
582 if (pTb->x86.fAttr == (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u)
583 {
584#ifdef VBOX_WITH_STATISTICS
585 pVCpu->iem.s.cTbLookupHits++;
586#endif
587 return pTb;
588 }
589 Log11(("TB miss: CS: %#x, wanted %#x\n", pTb->x86.fAttr, (uint16_t)pVCpu->cpum.GstCtx.cs.Attr.u));
590 }
591 else
592 Log11(("TB miss: fFlags: %#x, wanted %#x\n", pTb->fFlags, fFlags));
593 }
594 else
595 Log11(("TB miss: GCPhysPc: %#x, wanted %#x\n", pTb->GCPhysPc, GCPhysPc));
596
597 pTb = pTb->pNext;
598 }
599 RT_NOREF(pVM);
600 pVCpu->iem.s.cTbLookupMisses++;
601 return pTb;
602}
603
604
605static void iemThreadedTbAdd(PVMCC pVM, PVMCPUCC pVCpu, PIEMTB pTb)
606{
607 uint32_t const idxHash = IEMTBCACHE_HASH(&g_TbCache, pTb->fFlags, pTb->GCPhysPc);
608 pTb->pNext = g_TbCache.apHash[idxHash];
609 g_TbCache.apHash[idxHash] = pTb;
610 STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->iem.s.StatTbThreadedInstr, pTb->cInstructions);
611 STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->iem.s.StatTbThreadedCalls, pTb->Thrd.cCalls);
612 if (LogIs12Enabled())
613 {
614 Log12(("TB added: %p %RGp LB %#x fl=%#x idxHash=%#x cRanges=%u cInstr=%u cCalls=%u\n",
615 pTb, pTb->GCPhysPc, pTb->cbOpcodes, pTb->fFlags, idxHash, pTb->cRanges, pTb->cInstructions, pTb->Thrd.cCalls));
616 for (uint8_t idxRange = 0; idxRange < pTb->cRanges; idxRange++)
617 Log12((" range#%u: offPg=%#05x offOp=%#04x LB %#04x pg#%u=%RGp\n", idxRange, pTb->aRanges[idxRange].offPhysPage,
618 pTb->aRanges[idxRange].offOpcodes, pTb->aRanges[idxRange].cbOpcodes, pTb->aRanges[idxRange].idxPhysPage,
619 pTb->aRanges[idxRange].idxPhysPage == 0
620 ? pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK
621 : pTb->aGCPhysPages[pTb->aRanges[idxRange].idxPhysPage - 1]));
622 }
623 RT_NOREF(pVM);
624}
625
626
627/*
628 * Real code.
629 */
630
631#ifdef LOG_ENABLED
632/**
633 * Logs the current instruction.
634 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
635 * @param pszFunction The IEM function doing the execution.
636 */
637static void iemThreadedLogCurInstr(PVMCPUCC pVCpu, const char *pszFunction) RT_NOEXCEPT
638{
639# ifdef IN_RING3
640 if (LogIs2Enabled())
641 {
642 char szInstr[256];
643 uint32_t cbInstr = 0;
644 DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
645 DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
646 szInstr, sizeof(szInstr), &cbInstr);
647
648 PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
649 Log2(("**** %s fExec=%x pTb=%p\n"
650 " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
651 " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
652 " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
653 " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
654 " %s\n"
655 , pszFunction, pVCpu->iem.s.fExec, pVCpu->iem.s.pCurTbR3,
656 pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
657 pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
658 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
659 pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
660 pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
661 szInstr));
662
663 if (LogIs3Enabled())
664 DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
665 }
666 else
667# endif
668 LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
669 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
670}
671#endif /* LOG_ENABLED */
672
673
674static VBOXSTRICTRC iemThreadedCompileLongJumped(PVMCC pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
675{
676 RT_NOREF(pVM, pVCpu);
677 return rcStrict;
678}
679
680
681/**
682 * Initializes the decoder state when compiling TBs.
683 *
684 * This presumes that fExec has already be initialized.
685 *
686 * This is very similar to iemInitDecoder() and iemReInitDecoder(), so may need
687 * to apply fixes to them as well.
688 *
689 * @param pVCpu The cross context virtual CPU structure of the calling
690 * thread.
691 * @param fReInit Clear for the first call for a TB, set for subsequent calls
692 * from inside the compile loop where we can skip a couple of
693 * things.
694 */
695DECL_FORCE_INLINE(void) iemThreadedCompileInitDecoder(PVMCPUCC pVCpu, bool const fReInit)
696{
697 /* ASSUMES: That iemInitExec was already called and that anyone changing
698 CPU state affecting the fExec bits since then will have updated fExec! */
699 AssertMsg((pVCpu->iem.s.fExec & ~IEM_F_USER_OPTS) == iemCalcExecFlags(pVCpu),
700 ("fExec=%#x iemCalcExecModeFlags=%#x\n", pVCpu->iem.s.fExec, iemCalcExecFlags(pVCpu)));
701
702 IEMMODE const enmMode = IEM_GET_CPU_MODE(pVCpu);
703
704 /* Decoder state: */
705 pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
706 pVCpu->iem.s.enmEffAddrMode = enmMode;
707 if (enmMode != IEMMODE_64BIT)
708 {
709 pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
710 pVCpu->iem.s.enmEffOpSize = enmMode;
711 }
712 else
713 {
714 pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
715 pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
716 }
717 pVCpu->iem.s.fPrefixes = 0;
718 pVCpu->iem.s.uRexReg = 0;
719 pVCpu->iem.s.uRexB = 0;
720 pVCpu->iem.s.uRexIndex = 0;
721 pVCpu->iem.s.idxPrefix = 0;
722 pVCpu->iem.s.uVex3rdReg = 0;
723 pVCpu->iem.s.uVexLength = 0;
724 pVCpu->iem.s.fEvexStuff = 0;
725 pVCpu->iem.s.iEffSeg = X86_SREG_DS;
726 pVCpu->iem.s.offModRm = 0;
727 pVCpu->iem.s.iNextMapping = 0;
728
729 if (!fReInit)
730 {
731 pVCpu->iem.s.cActiveMappings = 0;
732 pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
733 pVCpu->iem.s.fEndTb = false;
734 pVCpu->iem.s.fTbCheckOpcodes = false;
735 pVCpu->iem.s.fTbBranched = IEMBRANCHED_F_NO;
736 pVCpu->iem.s.fTbCrossedPage = false;
737 }
738 else
739 {
740 Assert(pVCpu->iem.s.cActiveMappings == 0);
741 Assert(pVCpu->iem.s.rcPassUp == VINF_SUCCESS);
742 Assert(pVCpu->iem.s.fEndTb == false);
743 Assert(pVCpu->iem.s.fTbCrossedPage == false);
744 }
745
746#ifdef DBGFTRACE_ENABLED
747 switch (IEM_GET_CPU_MODE(pVCpu))
748 {
749 case IEMMODE_64BIT:
750 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.rip);
751 break;
752 case IEMMODE_32BIT:
753 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
754 break;
755 case IEMMODE_16BIT:
756 RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", IEM_GET_CPL(pVCpu), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
757 break;
758 }
759#endif
760}
761
762
763/**
764 * Initializes the opcode fetcher when starting the compilation.
765 *
766 * @param pVCpu The cross context virtual CPU structure of the calling
767 * thread.
768 */
769DECL_FORCE_INLINE(void) iemThreadedCompileInitOpcodeFetching(PVMCPUCC pVCpu)
770{
771 /* Almost everything is done by iemGetPcWithPhysAndCode() already. We just need to initialize the index into abOpcode. */
772#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
773 pVCpu->iem.s.offOpcode = 0;
774#else
775 RT_NOREF(pVCpu);
776#endif
777}
778
779
780/**
781 * Re-initializes the opcode fetcher between instructions while compiling.
782 *
783 * @param pVCpu The cross context virtual CPU structure of the calling
784 * thread.
785 */
786DECL_FORCE_INLINE(void) iemThreadedCompileReInitOpcodeFetching(PVMCPUCC pVCpu)
787{
788 if (pVCpu->iem.s.pbInstrBuf)
789 {
790 uint64_t off = pVCpu->cpum.GstCtx.rip;
791 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
792 off += pVCpu->cpum.GstCtx.cs.u64Base;
793 off -= pVCpu->iem.s.uInstrBufPc;
794 if (off < pVCpu->iem.s.cbInstrBufTotal)
795 {
796 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
797 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
798 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
799 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
800 else
801 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
802 }
803 else
804 {
805 pVCpu->iem.s.pbInstrBuf = NULL;
806 pVCpu->iem.s.offInstrNextByte = 0;
807 pVCpu->iem.s.offCurInstrStart = 0;
808 pVCpu->iem.s.cbInstrBuf = 0;
809 pVCpu->iem.s.cbInstrBufTotal = 0;
810 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
811 }
812 }
813 else
814 {
815 pVCpu->iem.s.offInstrNextByte = 0;
816 pVCpu->iem.s.offCurInstrStart = 0;
817 pVCpu->iem.s.cbInstrBuf = 0;
818 pVCpu->iem.s.cbInstrBufTotal = 0;
819#ifdef VBOX_STRICT
820 pVCpu->iem.s.GCPhysInstrBuf = NIL_RTGCPHYS;
821#endif
822 }
823#ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
824 pVCpu->iem.s.offOpcode = 0;
825#endif
826}
827
828
829DECLINLINE(void) iemThreadedCopyOpcodeBytesInline(PCVMCPUCC pVCpu, uint8_t *pbDst, uint8_t cbInstr)
830{
831 switch (cbInstr)
832 {
833 default: AssertMsgFailed(("%#x\n", cbInstr)); RT_FALL_THROUGH();
834 case 15: pbDst[14] = pVCpu->iem.s.abOpcode[14]; RT_FALL_THROUGH();
835 case 14: pbDst[13] = pVCpu->iem.s.abOpcode[13]; RT_FALL_THROUGH();
836 case 13: pbDst[12] = pVCpu->iem.s.abOpcode[12]; RT_FALL_THROUGH();
837 case 12: pbDst[11] = pVCpu->iem.s.abOpcode[11]; RT_FALL_THROUGH();
838 case 11: pbDst[10] = pVCpu->iem.s.abOpcode[10]; RT_FALL_THROUGH();
839 case 10: pbDst[9] = pVCpu->iem.s.abOpcode[9]; RT_FALL_THROUGH();
840 case 9: pbDst[8] = pVCpu->iem.s.abOpcode[8]; RT_FALL_THROUGH();
841 case 8: pbDst[7] = pVCpu->iem.s.abOpcode[7]; RT_FALL_THROUGH();
842 case 7: pbDst[6] = pVCpu->iem.s.abOpcode[6]; RT_FALL_THROUGH();
843 case 6: pbDst[5] = pVCpu->iem.s.abOpcode[5]; RT_FALL_THROUGH();
844 case 5: pbDst[4] = pVCpu->iem.s.abOpcode[4]; RT_FALL_THROUGH();
845 case 4: pbDst[3] = pVCpu->iem.s.abOpcode[3]; RT_FALL_THROUGH();
846 case 3: pbDst[2] = pVCpu->iem.s.abOpcode[2]; RT_FALL_THROUGH();
847 case 2: pbDst[1] = pVCpu->iem.s.abOpcode[1]; RT_FALL_THROUGH();
848 case 1: pbDst[0] = pVCpu->iem.s.abOpcode[0]; break;
849 }
850}
851
852
853/**
854 * Called by IEM_MC2_BEGIN_EMIT_CALLS() under one of these conditions:
855 *
856 * - CS LIM check required.
857 * - Must recheck opcode bytes.
858 * - Previous instruction branched.
859 * - TLB load detected, probably due to page crossing.
860 *
861 * @returns true if everything went well, false if we're out of space in the TB
862 * (e.g. opcode ranges).
863 * @param pVCpu The cross context virtual CPU structure of the calling
864 * thread.
865 * @param pTb The translation block being compiled.
866 */
867bool iemThreadedCompileBeginEmitCallsComplications(PVMCPUCC pVCpu, PIEMTB pTb)
868{
869 Assert((pVCpu->iem.s.GCPhysInstrBuf & GUEST_PAGE_OFFSET_MASK) == 0);
870 if (pVCpu->cpum.GstCtx.rip >= 0xc0000000 && !LogIsEnabled())
871 RTLogChangeFlags(NULL, 0, RTLOGFLAGS_DISABLED);
872
873 /*
874 * Prepare call now, even before we know if can accept the instruction in this TB.
875 * This allows us amending parameters w/o making every case suffer.
876 */
877 uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu);
878 uint16_t const offOpcode = pTb->cbOpcodes;
879 uint8_t idxRange = pTb->cRanges - 1;
880
881 PIEMTHRDEDCALLENTRY const pCall = &pTb->Thrd.paCalls[pTb->Thrd.cCalls];
882 pCall->offOpcode = offOpcode;
883 pCall->idxRange = idxRange;
884 pCall->cbOpcode = cbInstr;
885 pCall->auParams[0] = cbInstr;
886 pCall->auParams[1] = idxRange;
887 pCall->auParams[2] = offOpcode - pTb->aRanges[idxRange].offOpcodes;
888
889/** @todo check if we require IEMTB_F_CS_LIM_CHECKS for any new page we've
890 * gotten onto. If we do, stop */
891
892 /*
893 * Case 1: We've branched (RIP changed).
894 *
895 * Sub-case 1a: Same page, no TLB load (fTbCrossedPage is false).
896 * Req: 1 extra range, no extra phys.
897 *
898 * Sub-case 1b: Different page but no page boundrary crossing, so TLB load
899 * necessary (fTbCrossedPage is true).
900 * Req: 1 extra range, probably 1 extra phys page entry.
901 *
902 * Sub-case 1c: Different page, so TLB load necessary (fTbCrossedPage is true),
903 * but in addition we cross into the following page and require
904 * another TLB load.
905 * Req: 2 extra ranges, probably 2 extra phys page entries.
906 *
907 * Sub-case 1d: Same page, so no initial TLB load necessary, but we cross into
908 * the following page (thus fTbCrossedPage is true).
909 * Req: 2 extra ranges, probably 1 extra phys page entry.
910 *
911 * Note! The setting fTbCrossedPage is done by the iemOpcodeFetchBytesJmp, but
912 * it may trigger "spuriously" from the CPU point of view because of
913 * physical page changes that'll invalid the physical TLB and trigger a
914 * call to the function. In theory this be a big deal, just a bit
915 * performance loss as we'll pick the LoadingTlb variants.
916 *
917 * Note! We do not currently optimize branching to the next instruction (sorry
918 * 32-bit PIC code). We could maybe do that in the branching code that
919 * sets (or not) fTbBranched.
920 */
921 /** @todo Optimize 'jmp .next_instr' and 'call .next_instr'. Seen the jmp
922 * variant in win 3.1 code and the call variant in 32-bit linux PIC
923 * code. This'll require filtering out far jmps and calls, as they
924 * load CS which should technically be considered indirect since the
925 * GDT/LDT entry's base address can be modified independently from
926 * the code. */
927 if (pVCpu->iem.s.fTbBranched != 0)
928 {
929 if ( !pVCpu->iem.s.fTbCrossedPage /* 1a */
930 || pVCpu->iem.s.offCurInstrStart >= 0 /* 1b */ )
931 {
932 /* 1a + 1b - instruction fully within the branched to page. */
933 Assert(pVCpu->iem.s.offCurInstrStart >= 0);
934 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr <= GUEST_PAGE_SIZE);
935
936 if (!(pVCpu->iem.s.fTbBranched & IEMBRANCHED_F_ZERO))
937 {
938 /* Check that we've got a free range. */
939 idxRange += 1;
940 if (idxRange < RT_ELEMENTS(pTb->aRanges))
941 { /* likely */ }
942 else
943 {
944 Log8(("%04x:%08RX64: out of ranges after branch\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
945 return false;
946 }
947 pCall->idxRange = idxRange;
948 pCall->auParams[1] = idxRange;
949 pCall->auParams[2] = 0;
950
951 /* Check that we've got a free page slot. */
952 AssertCompile(RT_ELEMENTS(pTb->aGCPhysPages) == 2);
953 RTGCPHYS const GCPhysNew = pVCpu->iem.s.GCPhysInstrBuf & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
954 if ((pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysNew)
955 pTb->aRanges[idxRange].idxPhysPage = 0;
956 else if ( pTb->aGCPhysPages[0] == NIL_RTGCPHYS
957 || pTb->aGCPhysPages[0] == GCPhysNew)
958 {
959 pTb->aGCPhysPages[0] = GCPhysNew;
960 pTb->aRanges[idxRange].idxPhysPage = 1;
961 }
962 else if ( pTb->aGCPhysPages[1] == NIL_RTGCPHYS
963 || pTb->aGCPhysPages[1] == GCPhysNew)
964 {
965 pTb->aGCPhysPages[1] = GCPhysNew;
966 pTb->aRanges[idxRange].idxPhysPage = 2;
967 }
968 else
969 {
970 Log8(("%04x:%08RX64: out of aGCPhysPages entires after branch\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
971 return false;
972 }
973
974 /* Finish setting up the new range. */
975 pTb->aRanges[idxRange].offPhysPage = pVCpu->iem.s.offCurInstrStart;
976 pTb->aRanges[idxRange].offOpcodes = offOpcode;
977 pTb->aRanges[idxRange].cbOpcodes = cbInstr;
978 pTb->aRanges[idxRange].u2Unused = 0;
979 pTb->cRanges++;
980 }
981 else
982 {
983 Log8(("%04x:%08RX64: zero byte jump\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
984 pTb->aRanges[idxRange].cbOpcodes += cbInstr;
985 }
986
987 /* Determin which function we need to load & check.
988 Note! For jumps to a new page, we'll set both fTbBranched and
989 fTbCrossedPage to avoid unnecessary TLB work for intra
990 page branching */
991 if ( (pVCpu->iem.s.fTbBranched & (IEMBRANCHED_F_INDIRECT | IEMBRANCHED_F_FAR)) /* Far is basically indirect. */
992 || pVCpu->iem.s.fTbCrossedPage)
993 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
994 ? kIemThreadedFunc_CheckCsLimAndOpcodesLoadingTlb
995 : kIemThreadedFunc_CheckOpcodesLoadingTlb;
996 else if (pVCpu->iem.s.fTbBranched & (IEMBRANCHED_F_CONDITIONAL | /* paranoia: */ IEMBRANCHED_F_DIRECT))
997 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
998 ? kIemThreadedFunc_CheckCsLimAndPcAndOpcodes
999 : kIemThreadedFunc_CheckPcAndOpcodes;
1000 else
1001 {
1002 Assert(pVCpu->iem.s.fTbBranched & IEMBRANCHED_F_RELATIVE);
1003 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1004 ? kIemThreadedFunc_CheckCsLimAndOpcodes
1005 : kIemThreadedFunc_CheckOpcodes;
1006 }
1007 }
1008 else
1009 {
1010 /* 1c + 1d - instruction crosses pages. */
1011 Assert(pVCpu->iem.s.offCurInstrStart < 0);
1012 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr > 0);
1013
1014 /* Lazy bird: Check that this isn't case 1c, since we've already
1015 load the first physical address. End the TB and
1016 make it a case 2b instead.
1017
1018 Hmm. Too much bother to detect, so just do the same
1019 with case 1d as well. */
1020#if 0 /** @todo get back to this later when we've got the actual branch code in
1021 * place. */
1022 uint8_t const cbStartPage = (uint8_t)-pVCpu->iem.s.offCurInstrStart;
1023
1024 /* Check that we've got two free ranges. */
1025 if (idxRange + 2 < RT_ELEMENTS(pTb->aRanges))
1026 { /* likely */ }
1027 else
1028 return false;
1029 idxRange += 1;
1030 pCall->idxRange = idxRange;
1031 pCall->auParams[1] = idxRange;
1032 pCall->auParams[2] = 0;
1033
1034 /* ... */
1035
1036#else
1037 Log8(("%04x:%08RX64: complicated post-branch condition, ending TB.\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1038 return false;
1039#endif
1040 }
1041 }
1042
1043 /*
1044 * Case 2: Page crossing.
1045 *
1046 * Sub-case 2a: The instruction starts on the first byte in the next page.
1047 *
1048 * Sub-case 2b: The instruction has opcode bytes in both the current and
1049 * following page.
1050 *
1051 * Both cases requires a new range table entry and probably a new physical
1052 * page entry. The difference is in which functions to emit and whether to
1053 * add bytes to the current range.
1054 */
1055 else if (pVCpu->iem.s.fTbCrossedPage)
1056 {
1057 /* Check that we've got a free range. */
1058 idxRange += 1;
1059 if (idxRange < RT_ELEMENTS(pTb->aRanges))
1060 { /* likely */ }
1061 else
1062 {
1063 Log8(("%04x:%08RX64: out of ranges while crossing page\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1064 return false;
1065 }
1066
1067 /* Check that we've got a free page slot. */
1068 AssertCompile(RT_ELEMENTS(pTb->aGCPhysPages) == 2);
1069 RTGCPHYS const GCPhysNew = pVCpu->iem.s.GCPhysInstrBuf & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
1070 if ((pTb->GCPhysPc & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK) == GCPhysNew)
1071 pTb->aRanges[idxRange].idxPhysPage = 0;
1072 else if ( pTb->aGCPhysPages[0] == NIL_RTGCPHYS
1073 || pTb->aGCPhysPages[0] == GCPhysNew)
1074 {
1075 pTb->aGCPhysPages[0] = GCPhysNew;
1076 pTb->aRanges[idxRange].idxPhysPage = 1;
1077 }
1078 else if ( pTb->aGCPhysPages[1] == NIL_RTGCPHYS
1079 || pTb->aGCPhysPages[1] == GCPhysNew)
1080 {
1081 pTb->aGCPhysPages[1] = GCPhysNew;
1082 pTb->aRanges[idxRange].idxPhysPage = 2;
1083 }
1084 else
1085 {
1086 Log8(("%04x:%08RX64: out of aGCPhysPages entires while crossing page\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
1087 return false;
1088 }
1089
1090 if (((pTb->aRanges[idxRange - 1].offPhysPage + pTb->aRanges[idxRange - 1].cbOpcodes) & GUEST_PAGE_OFFSET_MASK) == 0)
1091 {
1092 Assert(pVCpu->iem.s.offCurInstrStart == 0);
1093 pCall->idxRange = idxRange;
1094 pCall->auParams[1] = idxRange;
1095 pCall->auParams[2] = 0;
1096
1097 /* Finish setting up the new range. */
1098 pTb->aRanges[idxRange].offPhysPage = pVCpu->iem.s.offCurInstrStart;
1099 pTb->aRanges[idxRange].offOpcodes = offOpcode;
1100 pTb->aRanges[idxRange].cbOpcodes = cbInstr;
1101 pTb->aRanges[idxRange].u2Unused = 0;
1102 pTb->cRanges++;
1103
1104 /* Determin which function we need to load & check. */
1105 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1106 ? kIemThreadedFunc_CheckCsLimAndOpcodesOnNewPageLoadingTlb
1107 : kIemThreadedFunc_CheckOpcodesOnNewPageLoadingTlb;
1108 }
1109 else
1110 {
1111 Assert(pVCpu->iem.s.offCurInstrStart < 0);
1112 Assert(pVCpu->iem.s.offCurInstrStart + cbInstr > 0);
1113 uint8_t const cbStartPage = (uint8_t)-pVCpu->iem.s.offCurInstrStart;
1114 pCall->auParams[0] |= (uint64_t)cbStartPage << 32;
1115
1116 /* We've good. Split the instruction over the old and new range table entries. */
1117 pTb->aRanges[idxRange - 1].cbOpcodes += cbStartPage;
1118
1119 pTb->aRanges[idxRange].offPhysPage = 0;
1120 pTb->aRanges[idxRange].offOpcodes = offOpcode + cbStartPage;
1121 pTb->aRanges[idxRange].cbOpcodes = cbInstr - cbStartPage;
1122 pTb->aRanges[idxRange].u2Unused = 0;
1123 pTb->cRanges++;
1124
1125 /* Determin which function we need to load & check. */
1126 if (pVCpu->iem.s.fTbCheckOpcodes)
1127 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1128 ? kIemThreadedFunc_CheckCsLimAndOpcodesAcrossPageLoadingTlb
1129 : kIemThreadedFunc_CheckOpcodesAcrossPageLoadingTlb;
1130 else
1131 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1132 ? kIemThreadedFunc_CheckCsLimAndOpcodesOnNextPageLoadingTlb
1133 : kIemThreadedFunc_CheckOpcodesOnNextPageLoadingTlb;
1134 }
1135 }
1136
1137 /*
1138 * Regular case: No new range required.
1139 */
1140 else
1141 {
1142 Assert(pVCpu->iem.s.fTbCheckOpcodes || (pTb->fFlags & IEMTB_F_CS_LIM_CHECKS));
1143 if (pVCpu->iem.s.fTbCheckOpcodes)
1144 pCall->enmFunction = pTb->fFlags & IEMTB_F_CS_LIM_CHECKS
1145 ? kIemThreadedFunc_CheckCsLimAndOpcodes
1146 : kIemThreadedFunc_CheckOpcodes;
1147 else
1148 pCall->enmFunction = kIemThreadedFunc_CheckCsLim;
1149
1150 iemThreadedCopyOpcodeBytesInline(pVCpu, &pTb->pabOpcodes[offOpcode], cbInstr);
1151 pTb->cbOpcodes = offOpcode + cbInstr;
1152 pTb->aRanges[idxRange].cbOpcodes += cbInstr;
1153 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1154 }
1155
1156 /*
1157 * Commit the call.
1158 */
1159 pTb->Thrd.cCalls++;
1160
1161 /*
1162 * Clear state.
1163 */
1164 pVCpu->iem.s.fTbBranched = IEMBRANCHED_F_NO;
1165 pVCpu->iem.s.fTbCrossedPage = false;
1166 pVCpu->iem.s.fTbCheckOpcodes = false;
1167
1168 /*
1169 * Copy opcode bytes.
1170 */
1171 iemThreadedCopyOpcodeBytesInline(pVCpu, &pTb->pabOpcodes[offOpcode], cbInstr);
1172 pTb->cbOpcodes = offOpcode + cbInstr;
1173 Assert(pTb->cbOpcodes <= pTb->cbOpcodesAllocated);
1174
1175 return true;
1176}
1177
1178
1179
1180/**
1181 * Compiles a new TB and executes it.
1182 *
1183 * We combine compilation and execution here as it makes it simpler code flow
1184 * in the main loop and it allows interpreting while compiling if we want to
1185 * explore that option.
1186 *
1187 * @returns Strict VBox status code.
1188 * @param pVM The cross context virtual machine structure.
1189 * @param pVCpu The cross context virtual CPU structure of the calling
1190 * thread.
1191 * @param GCPhysPc The physical address corresponding to the current
1192 * RIP+CS.BASE.
1193 * @param fExtraFlags Extra translation block flags: IEMTB_F_TYPE_THREADED and
1194 * maybe IEMTB_F_RIP_CHECKS.
1195 */
1196static VBOXSTRICTRC iemThreadedCompile(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysPc, uint32_t fExtraFlags) IEM_NOEXCEPT_MAY_LONGJMP
1197{
1198 /*
1199 * Allocate a new translation block.
1200 */
1201 PIEMTB pTb = iemThreadedTbAlloc(pVM, pVCpu, GCPhysPc, fExtraFlags | IEMTB_F_STATE_COMPILING);
1202 AssertReturn(pTb, VERR_IEM_TB_ALLOC_FAILED);
1203
1204 /* Set the current TB so iemThreadedCompileLongJumped and the CIMPL
1205 functions may get at it. */
1206 pVCpu->iem.s.pCurTbR3 = pTb;
1207
1208 /*
1209 * Now for the recomplication. (This mimicks IEMExecLots in many ways.)
1210 */
1211 iemThreadedCompileInitDecoder(pVCpu, false /*fReInit*/);
1212 iemThreadedCompileInitOpcodeFetching(pVCpu);
1213 VBOXSTRICTRC rcStrict;
1214 for (;;)
1215 {
1216 /* Process the next instruction. */
1217#ifdef LOG_ENABLED
1218 iemThreadedLogCurInstr(pVCpu, "CC");
1219 uint16_t const uCsLog = pVCpu->cpum.GstCtx.cs.Sel;
1220 uint64_t const uRipLog = pVCpu->cpum.GstCtx.rip;
1221#endif
1222 uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
1223 uint16_t const cCallsPrev = pTb->Thrd.cCalls;
1224
1225 rcStrict = FNIEMOP_CALL(g_apfnIemThreadedRecompilerOneByteMap[b]);
1226 if ( rcStrict == VINF_SUCCESS
1227 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS
1228 && !pVCpu->iem.s.fEndTb)
1229 {
1230 Assert(pTb->Thrd.cCalls > cCallsPrev);
1231 Assert(cCallsPrev - pTb->Thrd.cCalls < 5);
1232
1233 pVCpu->iem.s.cInstructions++;
1234 }
1235 else
1236 {
1237 Log8(("%04x:%08RX64: End TB - %u instr, %u calls, rc=%d\n",
1238 uCsLog, uRipLog, pTb->cInstructions, pTb->Thrd.cCalls, VBOXSTRICTRC_VAL(rcStrict)));
1239 if (rcStrict == VINF_IEM_RECOMPILE_END_TB)
1240 rcStrict = VINF_SUCCESS;
1241
1242 if (pTb->Thrd.cCalls > 0)
1243 {
1244 if (cCallsPrev != pTb->Thrd.cCalls)
1245 pVCpu->iem.s.cInstructions++;
1246 break;
1247 }
1248
1249 pVCpu->iem.s.pCurTbR3 = NULL;
1250 iemThreadedTbFree(pVM, pVCpu, pTb);
1251 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1252 }
1253
1254 /* Still space in the TB? */
1255 if ( pTb->Thrd.cCalls + 5 < pTb->Thrd.cAllocated
1256 && pTb->cbOpcodes + 16 <= pTb->cbOpcodesAllocated)
1257 iemThreadedCompileInitDecoder(pVCpu, true /*fReInit*/);
1258 else
1259 {
1260 Log8(("%04x:%08RX64: End TB - %u instr, %u calls, %u opcode bytes - full\n",
1261 uCsLog, uRipLog, pTb->cInstructions, pTb->Thrd.cCalls, pTb->cbOpcodes));
1262 break;
1263 }
1264 iemThreadedCompileReInitOpcodeFetching(pVCpu);
1265 }
1266
1267 /*
1268 * Complete the TB and link it.
1269 */
1270 pTb->fFlags = (pTb->fFlags & ~IEMTB_F_STATE_MASK) | IEMTB_F_STATE_READY;
1271 iemThreadedTbAdd(pVM, pVCpu, pTb);
1272
1273#ifdef IEM_COMPILE_ONLY_MODE
1274 /*
1275 * Execute the translation block.
1276 */
1277#endif
1278
1279 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1280}
1281
1282
1283/**
1284 * Executes a translation block.
1285 *
1286 * @returns Strict VBox status code.
1287 * @param pVCpu The cross context virtual CPU structure of the calling
1288 * thread.
1289 * @param pTb The translation block to execute.
1290 */
1291static VBOXSTRICTRC iemThreadedTbExec(PVMCPUCC pVCpu, PIEMTB pTb) IEM_NOEXCEPT_MAY_LONGJMP
1292{
1293 /* Check the opcodes in the first page before starting execution. */
1294 Assert(!(pVCpu->iem.s.GCPhysInstrBuf & (RTGCPHYS)GUEST_PAGE_OFFSET_MASK));
1295 Assert(pTb->aRanges[0].cbOpcodes <= pVCpu->iem.s.cbInstrBufTotal - pVCpu->iem.s.offInstrNextByte);
1296 if (memcmp(pTb->pabOpcodes, &pVCpu->iem.s.pbInstrBuf[pTb->aRanges[0].offPhysPage], pTb->aRanges[0].cbOpcodes) == 0)
1297 { /* likely */ }
1298 else
1299 {
1300 Log7(("TB obsolete: %p GCPhys=%RGp\n", pTb, pTb->GCPhysPc));
1301 iemThreadedTbFree(pVCpu->pVMR3, pVCpu, pTb);
1302 return VINF_SUCCESS;
1303 }
1304
1305 /* Set the current TB so CIMPL function may get at it. */
1306 pVCpu->iem.s.pCurTbR3 = pTb;
1307 pVCpu->iem.s.cTbExec++;
1308
1309 /* The execution loop. */
1310 PCIEMTHRDEDCALLENTRY pCallEntry = pTb->Thrd.paCalls;
1311 uint32_t cCallsLeft = pTb->Thrd.cCalls;
1312 while (cCallsLeft-- > 0)
1313 {
1314#ifdef LOG_ENABLED
1315 iemThreadedLogCurInstr(pVCpu, "EX");
1316 Log9(("%04x:%08RX64: #%d - %d %s\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
1317 pTb->Thrd.cCalls - cCallsLeft - 1, pCallEntry->enmFunction, g_apszIemThreadedFunctions[pCallEntry->enmFunction]));
1318#endif
1319 VBOXSTRICTRC const rcStrict = g_apfnIemThreadedFunctions[pCallEntry->enmFunction](pVCpu,
1320 pCallEntry->auParams[0],
1321 pCallEntry->auParams[1],
1322 pCallEntry->auParams[2]);
1323
1324 if (RT_LIKELY( rcStrict == VINF_SUCCESS
1325 && pVCpu->iem.s.rcPassUp == VINF_SUCCESS /** @todo this isn't great. */))
1326 pCallEntry++;
1327 else
1328 {
1329 pVCpu->iem.s.pCurTbR3 = NULL;
1330
1331 /* Some status codes are just to get us out of this loop and
1332 continue in a different translation block. */
1333 if (rcStrict == VINF_IEM_REEXEC_MODE_CHANGED)
1334 return iemExecStatusCodeFiddling(pVCpu, VINF_SUCCESS);
1335 return iemExecStatusCodeFiddling(pVCpu, rcStrict);
1336 }
1337 }
1338
1339 pVCpu->iem.s.pCurTbR3 = NULL;
1340 return VINF_SUCCESS;
1341}
1342
1343
1344/**
1345 * This is called when the PC doesn't match the current pbInstrBuf.
1346 *
1347 * Upon return, we're ready for opcode fetching. But please note that
1348 * pbInstrBuf can be NULL iff the memory doesn't have readable backing (i.e.
1349 * MMIO or unassigned).
1350 */
1351static RTGCPHYS iemGetPcWithPhysAndCodeMissed(PVMCPUCC pVCpu)
1352{
1353 pVCpu->iem.s.pbInstrBuf = NULL;
1354 pVCpu->iem.s.offCurInstrStart = 0;
1355 pVCpu->iem.s.offInstrNextByte = 0;
1356 iemOpcodeFetchBytesJmp(pVCpu, 0, NULL);
1357 return pVCpu->iem.s.GCPhysInstrBuf + pVCpu->iem.s.offCurInstrStart;
1358}
1359
1360
1361/** @todo need private inline decl for throw/nothrow matching IEM_WITH_SETJMP? */
1362DECL_FORCE_INLINE_THROW(RTGCPHYS) iemGetPcWithPhysAndCode(PVMCPUCC pVCpu)
1363{
1364 /*
1365 * Set uCurTbStartPc to RIP and calc the effective PC.
1366 */
1367 uint64_t uPc = pVCpu->cpum.GstCtx.rip;
1368 pVCpu->iem.s.uCurTbStartPc = uPc;
1369 Assert(pVCpu->cpum.GstCtx.cs.u64Base == 0 || !IEM_IS_64BIT_CODE(pVCpu));
1370 uPc += pVCpu->cpum.GstCtx.cs.u64Base;
1371
1372 /*
1373 * Advance within the current buffer (PAGE) when possible.
1374 */
1375 if (pVCpu->iem.s.pbInstrBuf)
1376 {
1377 uint64_t off = uPc - pVCpu->iem.s.uInstrBufPc;
1378 if (off < pVCpu->iem.s.cbInstrBufTotal)
1379 {
1380 pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
1381 pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
1382 if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
1383 pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
1384 else
1385 pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
1386
1387 return pVCpu->iem.s.GCPhysInstrBuf + off;
1388 }
1389 }
1390 return iemGetPcWithPhysAndCodeMissed(pVCpu);
1391}
1392
1393
1394/**
1395 * Determines the extra IEMTB_F_XXX flags.
1396 *
1397 * @returns IEMTB_F_TYPE_THREADED and maybe IEMTB_F_RIP_CHECKS.
1398 * @param pVCpu The cross context virtual CPU structure of the calling
1399 * thread.
1400 */
1401DECL_FORCE_INLINE(uint32_t) iemGetTbFlagsForCurrentPc(PVMCPUCC pVCpu)
1402{
1403 /*
1404 * Return IEMTB_F_RIP_CHECKS if the current PC is invalid or if it is
1405 * likely to go invalid before the end of the translation block.
1406 */
1407 if (IEM_IS_64BIT_CODE(pVCpu))
1408 return IEMTB_F_TYPE_THREADED;
1409
1410 if (RT_LIKELY( pVCpu->cpum.GstCtx.eip < pVCpu->cpum.GstCtx.cs.u32Limit
1411 && pVCpu->cpum.GstCtx.eip - pVCpu->cpum.GstCtx.cs.u32Limit >= X86_PAGE_SIZE))
1412 return IEMTB_F_TYPE_THREADED;
1413
1414 return IEMTB_F_TYPE_THREADED | IEMTB_F_CS_LIM_CHECKS;
1415}
1416
1417
1418VMMDECL(VBOXSTRICTRC) IEMExecRecompilerThreaded(PVMCC pVM, PVMCPUCC pVCpu)
1419{
1420 /*
1421 * See if there is an interrupt pending in TRPM, inject it if we can.
1422 */
1423 if (!TRPMHasTrap(pVCpu))
1424 { /* likely */ }
1425 else
1426 {
1427 VBOXSTRICTRC rcStrict = iemExecInjectPendingTrap(pVCpu);
1428 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
1429 { /*likely */ }
1430 else
1431 return rcStrict;
1432 }
1433
1434 /*
1435 * Init the execution environment.
1436 */
1437 iemInitExec(pVCpu, 0 /*fExecOpts*/);
1438
1439 /*
1440 * Run-loop.
1441 *
1442 * If we're using setjmp/longjmp we combine all the catching here to avoid
1443 * having to call setjmp for each block we're executing.
1444 */
1445 for (;;)
1446 {
1447 PIEMTB pTb = NULL;
1448 VBOXSTRICTRC rcStrict;
1449 IEM_TRY_SETJMP(pVCpu, rcStrict)
1450 {
1451 uint32_t const cPollRate = 511; /* EM.cpp passes 4095 to IEMExecLots, so an eigth of that seems reasonable for now. */
1452 for (uint32_t iIterations = 0; ; iIterations++)
1453 {
1454 /* Translate PC to physical address, we'll need this for both lookup and compilation. */
1455 RTGCPHYS const GCPhysPc = iemGetPcWithPhysAndCode(pVCpu);
1456 uint32_t const fExtraFlags = iemGetTbFlagsForCurrentPc(pVCpu);
1457
1458 pTb = iemThreadedTbLookup(pVM, pVCpu, GCPhysPc, fExtraFlags);
1459 if (pTb)
1460 rcStrict = iemThreadedTbExec(pVCpu, pTb);
1461 else
1462 rcStrict = iemThreadedCompile(pVM, pVCpu, GCPhysPc, fExtraFlags);
1463 if (rcStrict == VINF_SUCCESS)
1464 {
1465 Assert(pVCpu->iem.s.cActiveMappings == 0);
1466
1467 uint64_t fCpu = pVCpu->fLocalForcedActions;
1468 fCpu &= VMCPU_FF_ALL_MASK & ~( VMCPU_FF_PGM_SYNC_CR3
1469 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
1470 | VMCPU_FF_TLB_FLUSH
1471 | VMCPU_FF_UNHALT );
1472 if (RT_LIKELY( ( !fCpu
1473 || ( !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
1474 && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
1475 && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
1476 {
1477 if (RT_LIKELY( (iIterations & cPollRate) != 0
1478 || !TMTimerPollBool(pVM, pVCpu)))
1479 {
1480
1481 }
1482 else
1483 return VINF_SUCCESS;
1484 }
1485 else
1486 return VINF_SUCCESS;
1487 }
1488 else
1489 return rcStrict;
1490 }
1491 }
1492 IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
1493 {
1494 pVCpu->iem.s.cLongJumps++;
1495 if (pVCpu->iem.s.cActiveMappings > 0)
1496 iemMemRollback(pVCpu);
1497
1498 /* If pTb isn't NULL we're in iemThreadedTbExec. */
1499 if (!pTb)
1500 {
1501 /* If pCurTbR3 is NULL, we're in iemGetPcWithPhysAndCode.*/
1502 pTb = pVCpu->iem.s.pCurTbR3;
1503 if (pTb)
1504 {
1505 /* If the pCurTbR3 block is in compiling state, we're in iemThreadedCompile,
1506 otherwise it's iemThreadedTbExec inside iemThreadedCompile (compile option). */
1507 if ((pTb->fFlags & IEMTB_F_STATE_MASK) == IEMTB_F_STATE_COMPILING)
1508 return iemThreadedCompileLongJumped(pVM, pVCpu, rcStrict);
1509 }
1510 }
1511 return rcStrict;
1512 }
1513 IEM_CATCH_LONGJMP_END(pVCpu);
1514 }
1515}
1516
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette