VirtualBox

source: vbox/trunk/src/VBox/VMM/include/PGMInternal.h@ 92679

Last change on this file since 92679 was 92626, checked in by vboxsync, 3 years ago

VMM: Nested VMX: bugref:10092 Adjust PGM APIs and translate nested-guest CR3 prior to mapping them when switching mode and other places.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 168.4 KB
Line 
1/* $Id: PGMInternal.h 92626 2021-11-29 12:32:58Z vboxsync $ */
2/** @file
3 * PGM - Internal header file.
4 */
5
6/*
7 * Copyright (C) 2006-2020 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18#ifndef VMM_INCLUDED_SRC_include_PGMInternal_h
19#define VMM_INCLUDED_SRC_include_PGMInternal_h
20#ifndef RT_WITHOUT_PRAGMA_ONCE
21# pragma once
22#endif
23
24#include <VBox/cdefs.h>
25#include <VBox/types.h>
26#include <VBox/err.h>
27#include <VBox/dbg.h>
28#include <VBox/vmm/stam.h>
29#include <VBox/param.h>
30#include <VBox/vmm/vmm.h>
31#include <VBox/vmm/mm.h>
32#include <VBox/vmm/pdmcritsect.h>
33#include <VBox/vmm/pdmapi.h>
34#include <VBox/dis.h>
35#include <VBox/vmm/dbgf.h>
36#include <VBox/log.h>
37#include <VBox/vmm/gmm.h>
38#include <VBox/vmm/hm.h>
39#include <iprt/asm.h>
40#include <iprt/assert.h>
41#include <iprt/avl.h>
42#include <iprt/critsect.h>
43#include <iprt/list-off32.h>
44#include <iprt/sha.h>
45
46
47
48/** @defgroup grp_pgm_int Internals
49 * @ingroup grp_pgm
50 * @internal
51 * @{
52 */
53
54
55/** @name PGM Compile Time Config
56 * @{
57 */
58
59/**
60 * Check and skip global PDEs for non-global flushes
61 */
62#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
63
64/**
65 * Optimization for PAE page tables that are modified often
66 */
67//#if 0 /* disabled again while debugging */
68#define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
69//#endif
70
71/**
72 * Large page support enabled only on 64 bits hosts; applies to nested paging only.
73 */
74#define PGM_WITH_LARGE_PAGES
75
76/**
77 * Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and
78 * VMX_EXIT_EPT_MISCONFIG.
79 */
80#define PGM_WITH_MMIO_OPTIMIZATIONS
81
82/**
83 * Sync N pages instead of a whole page table
84 */
85#define PGM_SYNC_N_PAGES
86
87/**
88 * Number of pages to sync during a page fault
89 *
90 * When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
91 * causes a lot of unnecessary extents and also is slower than taking more \#PFs.
92 *
93 * Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
94 * world switch overhead, so let's sync more.
95 */
96#ifdef IN_RING0
97/* Chose 32 based on the compile test in @bugref{4219}; 64 shows worse stats.
98 * 32 again shows better results than 16; slightly more overhead in the \#PF handler,
99 * but ~5% fewer faults.
100 */
101# define PGM_SYNC_NR_PAGES 32
102#else
103# define PGM_SYNC_NR_PAGES 8
104#endif
105
106/**
107 * Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
108 */
109#define PGM_MAX_PHYSCACHE_ENTRIES 64
110#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
111
112
113/** @def PGMPOOL_CFG_MAX_GROW
114 * The maximum number of pages to add to the pool in one go.
115 */
116#define PGMPOOL_CFG_MAX_GROW (_2M >> PAGE_SHIFT)
117
118/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
119 * Enables some extra assertions for virtual handlers (mainly phys2virt related).
120 */
121#ifdef VBOX_STRICT
122# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
123#endif
124
125/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
126 * Enables the experimental lazy page allocation code. */
127#ifdef DOXYGEN_RUNNING
128# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC
129#endif
130
131/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
132 * Enables real write monitoring of pages, i.e. mapping them read-only and
133 * only making them writable when getting a write access \#PF. */
134#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
135
136/** @def VBOX_WITH_PGM_NEM_MODE
137 * Enabled the NEM memory management mode in PGM. See PGM::fNemMode for
138 * details. */
139#ifdef DOXYGEN_RUNNING
140# define VBOX_WITH_PGM_NEM_MODE
141#endif
142
143/** @} */
144
145
146/** @name PDPT and PML4 flags.
147 * These are placed in the three bits available for system programs in
148 * the PDPT and PML4 entries.
149 * @{ */
150/** The entry is a permanent one and it's must always be present.
151 * Never free such an entry. */
152#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
153/** PGM specific bits in PML4 entries. */
154#define PGM_PML4_FLAGS 0
155/** PGM specific bits in PDPT entries. */
156#define PGM_PDPT_FLAGS (PGM_PLXFLAGS_PERMANENT)
157/** @} */
158
159/** @name Page directory flags.
160 * These are placed in the three bits available for system programs in
161 * the page directory entries.
162 * @{ */
163/** Indicates the original entry was a big page.
164 * @remarks This is currently only used for statistics and can be recycled. */
165#define PGM_PDFLAGS_BIG_PAGE RT_BIT_64(9)
166/** Made read-only to facilitate dirty bit tracking. */
167#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
168/** @} */
169
170/** @name Page flags.
171 * These are placed in the three bits available for system programs in
172 * the page entries.
173 * @{ */
174/** Made read-only to facilitate dirty bit tracking. */
175#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
176
177#ifndef PGM_PTFLAGS_CSAM_VALIDATED
178/** Scanned and approved by CSAM (tm).
179 * NOTE: Must be identical to the one defined in CSAMInternal.h!!
180 * @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/vmm/pgm.h. */
181#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
182#endif
183
184/** @} */
185
186/** @name Defines used to indicate the shadow and guest paging in the templates.
187 * @{ */
188#define PGM_TYPE_REAL 1
189#define PGM_TYPE_PROT 2
190#define PGM_TYPE_32BIT 3
191#define PGM_TYPE_PAE 4
192#define PGM_TYPE_AMD64 5
193#define PGM_TYPE_NESTED_32BIT 6
194#define PGM_TYPE_NESTED_PAE 7
195#define PGM_TYPE_NESTED_AMD64 8
196#define PGM_TYPE_EPT 9
197#define PGM_TYPE_NONE 10 /**< Dummy shadow paging mode for NEM. */
198#define PGM_TYPE_END (PGM_TYPE_NONE + 1)
199#define PGM_TYPE_FIRST_SHADOW PGM_TYPE_32BIT /**< The first type used by shadow paging. */
200/** @} */
201
202/** @name Defines used to indicate the second-level
203 * address translation (SLAT) modes in the templates.
204 * @{ */
205#define PGM_SLAT_TYPE_EPT (PGM_TYPE_END + 1)
206#define PGM_SLAT_TYPE_32BIT (PGM_TYPE_END + 2)
207#define PGM_SLAT_TYPE_PAE (PGM_TYPE_END + 3)
208#define PGM_SLAT_TYPE_AMD64 (PGM_TYPE_END + 4)
209/** @} */
210
211/** Macro for checking if the guest is using paging.
212 * @param uGstType PGM_TYPE_*
213 * @param uShwType PGM_TYPE_*
214 * @remark ASSUMES certain order of the PGM_TYPE_* values.
215 */
216#define PGM_WITH_PAGING(uGstType, uShwType) \
217 ( (uGstType) >= PGM_TYPE_32BIT \
218 && (uShwType) < PGM_TYPE_NESTED_32BIT)
219
220/** Macro for checking if the guest supports the NX bit.
221 * @param uGstType PGM_TYPE_*
222 * @param uShwType PGM_TYPE_*
223 * @remark ASSUMES certain order of the PGM_TYPE_* values.
224 */
225#define PGM_WITH_NX(uGstType, uShwType) \
226 ( (uGstType) >= PGM_TYPE_PAE \
227 && (uShwType) < PGM_TYPE_NESTED_32BIT)
228
229/** Macro for checking for nested.
230 * @param uType PGM_TYPE_*
231 */
232#define PGM_TYPE_IS_NESTED(uType) \
233 ( (uType) == PGM_TYPE_NESTED_32BIT \
234 || (uType) == PGM_TYPE_NESTED_PAE \
235 || (uType) == PGM_TYPE_NESTED_AMD64)
236
237/** Macro for checking for nested or EPT.
238 * @param uType PGM_TYPE_*
239 */
240#define PGM_TYPE_IS_NESTED_OR_EPT(uType) \
241 ( (uType) == PGM_TYPE_NESTED_32BIT \
242 || (uType) == PGM_TYPE_NESTED_PAE \
243 || (uType) == PGM_TYPE_NESTED_AMD64 \
244 || (uType) == PGM_TYPE_EPT)
245
246
247
248/** @def PGM_HCPHYS_2_PTR
249 * Maps a HC physical page pool address to a virtual address.
250 *
251 * @returns VBox status code.
252 * @param pVM The cross context VM structure.
253 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
254 * @param HCPhys The HC physical address to map to a virtual one.
255 * @param ppv Where to store the virtual address. No need to cast
256 * this.
257 *
258 * @remark There is no need to assert on the result.
259 */
260#define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) pgmPoolHCPhys2Ptr(pVM, HCPhys, (void **)(ppv))
261
262/** @def PGM_GCPHYS_2_PTR_V2
263 * Maps a GC physical page address to a virtual address.
264 *
265 * @returns VBox status code.
266 * @param pVM The cross context VM structure.
267 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
268 * @param GCPhys The GC physical address to map to a virtual one.
269 * @param ppv Where to store the virtual address. No need to cast this.
270 *
271 * @remark Use with care as we don't have so much dynamic mapping space in
272 * ring-0 on 32-bit darwin and in RC.
273 * @remark There is no need to assert on the result.
274 */
275#define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
276 pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
277
278/** @def PGM_GCPHYS_2_PTR
279 * Maps a GC physical page address to a virtual address.
280 *
281 * @returns VBox status code.
282 * @param pVM The cross context VM structure.
283 * @param GCPhys The GC physical address to map to a virtual one.
284 * @param ppv Where to store the virtual address. No need to cast this.
285 *
286 * @remark Use with care as we don't have so much dynamic mapping space in
287 * ring-0 on 32-bit darwin and in RC.
288 * @remark There is no need to assert on the result.
289 */
290#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv)
291
292/** @def PGM_GCPHYS_2_PTR_BY_VMCPU
293 * Maps a GC physical page address to a virtual address.
294 *
295 * @returns VBox status code.
296 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
297 * @param GCPhys The GC physical address to map to a virtual one.
298 * @param ppv Where to store the virtual address. No need to cast this.
299 *
300 * @remark Use with care as we don't have so much dynamic mapping space in
301 * ring-0 on 32-bit darwin and in RC.
302 * @remark There is no need to assert on the result.
303 */
304#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv)
305
306/** @def PGM_GCPHYS_2_PTR_EX
307 * Maps a unaligned GC physical page address to a virtual address.
308 *
309 * @returns VBox status code.
310 * @param pVM The cross context VM structure.
311 * @param GCPhys The GC physical address to map to a virtual one.
312 * @param ppv Where to store the virtual address. No need to cast this.
313 *
314 * @remark Use with care as we don't have so much dynamic mapping space in
315 * ring-0 on 32-bit darwin and in RC.
316 * @remark There is no need to assert on the result.
317 */
318#define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
319 pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
320
321/** @def PGM_DYNMAP_UNUSED_HINT
322 * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
323 * is no longer used.
324 *
325 * For best effect only apply this to the page that was mapped most recently.
326 *
327 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
328 * @param pvPage The pool page.
329 */
330#define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) do {} while (0)
331
332/** @def PGM_DYNMAP_UNUSED_HINT_VM
333 * Hints to the dynamic mapping code in RC and R0/darwin that the specified page
334 * is no longer used.
335 *
336 * For best effect only apply this to the page that was mapped most recently.
337 *
338 * @param pVM The cross context VM structure.
339 * @param pvPage The pool page.
340 */
341#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage) PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage)
342
343
344/** @def PGM_INVL_PG
345 * Invalidates a page.
346 *
347 * @param pVCpu The cross context virtual CPU structure.
348 * @param GCVirt The virtual address of the page to invalidate.
349 */
350#ifdef IN_RING0
351# define PGM_INVL_PG(pVCpu, GCVirt) HMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
352#elif defined(IN_RING3)
353# define PGM_INVL_PG(pVCpu, GCVirt) HMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
354#else
355# error "Not IN_RING0 or IN_RING3!"
356#endif
357
358/** @def PGM_INVL_PG_ALL_VCPU
359 * Invalidates a page on all VCPUs
360 *
361 * @param pVM The cross context VM structure.
362 * @param GCVirt The virtual address of the page to invalidate.
363 */
364#ifdef IN_RING0
365# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
366#else
367# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
368#endif
369
370/** @def PGM_INVL_BIG_PG
371 * Invalidates a 4MB page directory entry.
372 *
373 * @param pVCpu The cross context virtual CPU structure.
374 * @param GCVirt The virtual address within the page directory to invalidate.
375 */
376#ifdef IN_RING0
377# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HMFlushTlb(pVCpu)
378#else
379# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HMFlushTlb(pVCpu)
380#endif
381
382/** @def PGM_INVL_VCPU_TLBS()
383 * Invalidates the TLBs of the specified VCPU
384 *
385 * @param pVCpu The cross context virtual CPU structure.
386 */
387#ifdef IN_RING0
388# define PGM_INVL_VCPU_TLBS(pVCpu) HMFlushTlb(pVCpu)
389#else
390# define PGM_INVL_VCPU_TLBS(pVCpu) HMFlushTlb(pVCpu)
391#endif
392
393/** @def PGM_INVL_ALL_VCPU_TLBS()
394 * Invalidates the TLBs of all VCPUs
395 *
396 * @param pVM The cross context VM structure.
397 */
398#ifdef IN_RING0
399# define PGM_INVL_ALL_VCPU_TLBS(pVM) HMFlushTlbOnAllVCpus(pVM)
400#else
401# define PGM_INVL_ALL_VCPU_TLBS(pVM) HMFlushTlbOnAllVCpus(pVM)
402#endif
403
404
405/** @name Safer Shadow PAE PT/PTE
406 * For helping avoid misinterpreting invalid PAE/AMD64 page table entries as
407 * present.
408 *
409 * @{
410 */
411#if 1
412/**
413 * For making sure that u1Present and X86_PTE_P checks doesn't mistake
414 * invalid entries for present.
415 * @sa X86PTEPAE.
416 */
417typedef union PGMSHWPTEPAE
418{
419 /** Unsigned integer view */
420 X86PGPAEUINT uCareful;
421 /* Not other views. */
422} PGMSHWPTEPAE;
423
424# define PGMSHWPTEPAE_IS_P(Pte) ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P )
425# define PGMSHWPTEPAE_IS_RW(Pte) ( !!((Pte).uCareful & X86_PTE_RW))
426# define PGMSHWPTEPAE_IS_US(Pte) ( !!((Pte).uCareful & X86_PTE_US))
427# define PGMSHWPTEPAE_IS_A(Pte) ( !!((Pte).uCareful & X86_PTE_A))
428# define PGMSHWPTEPAE_IS_D(Pte) ( !!((Pte).uCareful & X86_PTE_D))
429# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) )
430# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).uCareful & (X86_PTE_P | X86_PTE_RW | X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P | X86_PTE_RW) )
431# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).uCareful )
432# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).uCareful & X86_PTE_PAE_PG_MASK )
433# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).uCareful ) /**< Use with care. */
434# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).uCareful = (uVal); } while (0)
435# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).uCareful = (Pte2).uCareful; } while (0)
436# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0)
437# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0)
438# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
439# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).uCareful |= X86_PTE_RW; } while (0)
440
441/**
442 * For making sure that u1Present and X86_PTE_P checks doesn't mistake
443 * invalid entries for present.
444 * @sa X86PTPAE.
445 */
446typedef struct PGMSHWPTPAE
447{
448 PGMSHWPTEPAE a[X86_PG_PAE_ENTRIES];
449} PGMSHWPTPAE;
450
451#else
452typedef X86PTEPAE PGMSHWPTEPAE;
453typedef X86PTPAE PGMSHWPTPAE;
454# define PGMSHWPTEPAE_IS_P(Pte) ( (Pte).n.u1Present )
455# define PGMSHWPTEPAE_IS_RW(Pte) ( (Pte).n.u1Write )
456# define PGMSHWPTEPAE_IS_US(Pte) ( (Pte).n.u1User )
457# define PGMSHWPTEPAE_IS_A(Pte) ( (Pte).n.u1Accessed )
458# define PGMSHWPTEPAE_IS_D(Pte) ( (Pte).n.u1Dirty )
459# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
460# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) )
461# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).u )
462# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).u & X86_PTE_PAE_PG_MASK )
463# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).u ) /**< Use with care. */
464# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).u = (uVal); } while (0)
465# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).u = (Pte2).u; } while (0)
466# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0)
467# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
468# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
469# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).u |= X86_PTE_RW; } while (0)
470
471#endif
472
473/** Pointer to a shadow PAE PTE. */
474typedef PGMSHWPTEPAE *PPGMSHWPTEPAE;
475/** Pointer to a const shadow PAE PTE. */
476typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE;
477
478/** Pointer to a shadow PAE page table. */
479typedef PGMSHWPTPAE *PPGMSHWPTPAE;
480/** Pointer to a const shadow PAE page table. */
481typedef PGMSHWPTPAE const *PCPGMSHWPTPAE;
482/** @} */
483
484
485/**
486 * Physical page access handler type registration.
487 */
488typedef struct PGMPHYSHANDLERTYPEINT
489{
490 /** Number of references. */
491 uint32_t volatile cRefs;
492 /** Magic number (PGMPHYSHANDLERTYPEINT_MAGIC). */
493 uint32_t u32Magic;
494 /** Link of handler types anchored in PGMTREES::HeadPhysHandlerTypes. */
495 RTLISTOFF32NODE ListNode;
496 /** The kind of accesses we're handling. */
497 PGMPHYSHANDLERKIND enmKind;
498 /** The PGM_PAGE_HNDL_PHYS_STATE_XXX value corresponding to enmKind. */
499 uint8_t uState;
500 /** Whether to keep the PGM lock when calling the handler. */
501 bool fKeepPgmLock;
502 bool afPadding[2];
503 /** Pointer to R3 callback function. */
504 R3PTRTYPE(PFNPGMPHYSHANDLER) pfnHandlerR3;
505 /** Pointer to R0 callback function. */
506 R0PTRTYPE(PFNPGMPHYSHANDLER) pfnHandlerR0;
507 /** Pointer to R0 callback function for \#PFs. */
508 R0PTRTYPE(PFNPGMRZPHYSPFHANDLER) pfnPfHandlerR0;
509 /** Description / Name. For easing debugging. */
510 R3PTRTYPE(const char *) pszDesc;
511} PGMPHYSHANDLERTYPEINT;
512/** Pointer to a physical access handler type registration. */
513typedef PGMPHYSHANDLERTYPEINT *PPGMPHYSHANDLERTYPEINT;
514/** Magic value for the physical handler callbacks (Robert A. Heinlein). */
515#define PGMPHYSHANDLERTYPEINT_MAGIC UINT32_C(0x19070707)
516/** Magic value for the physical handler callbacks. */
517#define PGMPHYSHANDLERTYPEINT_MAGIC_DEAD UINT32_C(0x19880508)
518
519/**
520 * Converts a handle to a pointer.
521 * @returns PPGMPHYSHANDLERTYPEINT
522 * @param a_pVM The cross context VM structure.
523 * @param a_hType Physical access handler type handle.
524 */
525#define PGMPHYSHANDLERTYPEINT_FROM_HANDLE(a_pVM, a_hType) ((PPGMPHYSHANDLERTYPEINT)MMHyperHeapOffsetToPtr(a_pVM, a_hType))
526
527
528/**
529 * Physical page access handler structure.
530 *
531 * This is used to keep track of physical address ranges
532 * which are being monitored in some kind of way.
533 */
534typedef struct PGMPHYSHANDLER
535{
536 AVLROGCPHYSNODECORE Core;
537 /** Number of pages to update. */
538 uint32_t cPages;
539 /** Set if we have pages that have been aliased. */
540 uint32_t cAliasedPages;
541 /** Set if we have pages that have temporarily been disabled. */
542 uint32_t cTmpOffPages;
543 /** Registered handler type handle (heap offset). */
544 PGMPHYSHANDLERTYPE hType;
545 /** User argument for R3 handlers. */
546 R3PTRTYPE(void *) pvUserR3;
547 /** User argument for R0 handlers. */
548 R0PTRTYPE(void *) pvUserR0;
549 /** Description / Name. For easing debugging. */
550 R3PTRTYPE(const char *) pszDesc;
551#ifdef VBOX_WITH_STATISTICS
552 /** Profiling of this handler. */
553 STAMPROFILE Stat;
554#endif
555} PGMPHYSHANDLER;
556/** Pointer to a physical page access handler structure. */
557typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
558
559/**
560 * Gets the type record for a physical handler (no reference added).
561 * @returns PPGMPHYSHANDLERTYPEINT
562 * @param a_pVM The cross context VM structure.
563 * @param a_pPhysHandler Pointer to the physical handler structure
564 * (PGMPHYSHANDLER).
565 */
566#define PGMPHYSHANDLER_GET_TYPE(a_pVM, a_pPhysHandler) PGMPHYSHANDLERTYPEINT_FROM_HANDLE(a_pVM, (a_pPhysHandler)->hType)
567
568
569/**
570 * A Physical Guest Page tracking structure.
571 *
572 * The format of this structure is complicated because we have to fit a lot
573 * of information into as few bits as possible. The format is also subject
574 * to change (there is one coming up soon). Which means that for we'll be
575 * using PGM_PAGE_GET_*, PGM_PAGE_IS_ and PGM_PAGE_SET_* macros for *all*
576 * accesses to the structure.
577 */
578typedef union PGMPAGE
579{
580 /** Structured view. */
581 struct
582 {
583 /** 1:0 - The physical handler state (PGM_PAGE_HNDL_PHYS_STATE_*). */
584 uint64_t u2HandlerPhysStateY : 2;
585 /** 3:2 - Paging structure needed to map the page
586 * (PGM_PAGE_PDE_TYPE_*). */
587 uint64_t u2PDETypeY : 2;
588 /** 4 - Unused (was used by FTE for dirty tracking). */
589 uint64_t fUnused1 : 1;
590 /** 5 - Flag indicating that a write monitored page was written to
591 * when set. */
592 uint64_t fWrittenToY : 1;
593 /** 7:6 - Unused. */
594 uint64_t u2Unused0 : 2;
595 /** 9:8 - Unused (was used by PGM_PAGE_HNDL_VIRT_STATE_*). */
596 uint64_t u2Unused1 : 2;
597 /** 11:10 - NEM state bits. */
598 uint64_t u2NemStateY : 2;
599 /** 12:48 - The host physical frame number (shift left to get the
600 * address). */
601 uint64_t HCPhysFN : 36;
602 /** 50:48 - The page state. */
603 uint64_t uStateY : 3;
604 /** 51:53 - The page type (PGMPAGETYPE). */
605 uint64_t uTypeY : 3;
606 /** 63:54 - PTE index for usage tracking (page pool). */
607 uint64_t u10PteIdx : 10;
608
609 /** The GMM page ID.
610 * @remarks In the current implementation, MMIO2 and pages aliased to
611 * MMIO2 pages will be exploiting this field to calculate the
612 * ring-3 mapping address corresponding to the page.
613 * Later we may consider including MMIO2 management into GMM. */
614 uint32_t idPage;
615 /** Usage tracking (page pool). */
616 uint16_t u16TrackingY;
617 /** The number of read locks on this page. */
618 uint8_t cReadLocksY;
619 /** The number of write locks on this page. */
620 uint8_t cWriteLocksY;
621 } s;
622
623 /** 64-bit integer view. */
624 uint64_t au64[2];
625 /** 16-bit view. */
626 uint32_t au32[4];
627 /** 16-bit view. */
628 uint16_t au16[8];
629 /** 8-bit view. */
630 uint8_t au8[16];
631} PGMPAGE;
632AssertCompileSize(PGMPAGE, 16);
633/** Pointer to a physical guest page. */
634typedef PGMPAGE *PPGMPAGE;
635/** Pointer to a const physical guest page. */
636typedef const PGMPAGE *PCPGMPAGE;
637/** Pointer to a physical guest page pointer. */
638typedef PPGMPAGE *PPPGMPAGE;
639
640
641/**
642 * Clears the page structure.
643 * @param a_pPage Pointer to the physical guest page tracking structure.
644 */
645#define PGM_PAGE_CLEAR(a_pPage) \
646 do { \
647 (a_pPage)->au64[0] = 0; \
648 (a_pPage)->au64[1] = 0; \
649 } while (0)
650
651/**
652 * Initializes the page structure.
653 * @param a_pPage Pointer to the physical guest page tracking structure.
654 * @param a_HCPhys The host physical address of the page.
655 * @param a_idPage The (GMM) page ID of the page.
656 * @param a_uType The page type (PGMPAGETYPE).
657 * @param a_uState The page state (PGM_PAGE_STATE_XXX).
658 */
659#define PGM_PAGE_INIT(a_pPage, a_HCPhys, a_idPage, a_uType, a_uState) \
660 do { \
661 RTHCPHYS SetHCPhysTmp = (a_HCPhys); \
662 AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
663 (a_pPage)->au64[0] = SetHCPhysTmp; \
664 (a_pPage)->au64[1] = 0; \
665 (a_pPage)->s.idPage = (a_idPage); \
666 (a_pPage)->s.uStateY = (a_uState); \
667 (a_pPage)->s.uTypeY = (a_uType); \
668 } while (0)
669
670/**
671 * Initializes the page structure of a ZERO page.
672 * @param a_pPage Pointer to the physical guest page tracking structure.
673 * @param a_pVM The VM handle (for getting the zero page address).
674 * @param a_uType The page type (PGMPAGETYPE).
675 */
676#define PGM_PAGE_INIT_ZERO(a_pPage, a_pVM, a_uType) \
677 PGM_PAGE_INIT((a_pPage), (a_pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (a_uType), PGM_PAGE_STATE_ZERO)
678
679
680/** @name The Page state, PGMPAGE::uStateY.
681 * @{ */
682/** The zero page.
683 * This is a per-VM page that's never ever mapped writable. */
684#define PGM_PAGE_STATE_ZERO 0U
685/** A allocated page.
686 * This is a per-VM page allocated from the page pool (or wherever
687 * we get MMIO2 pages from if the type is MMIO2).
688 */
689#define PGM_PAGE_STATE_ALLOCATED 1U
690/** A allocated page that's being monitored for writes.
691 * The shadow page table mappings are read-only. When a write occurs, the
692 * fWrittenTo member is set, the page remapped as read-write and the state
693 * moved back to allocated. */
694#define PGM_PAGE_STATE_WRITE_MONITORED 2U
695/** The page is shared, aka. copy-on-write.
696 * This is a page that's shared with other VMs. */
697#define PGM_PAGE_STATE_SHARED 3U
698/** The page is ballooned, so no longer available for this VM. */
699#define PGM_PAGE_STATE_BALLOONED 4U
700/** @} */
701
702
703/** Asserts lock ownership in some of the PGM_PAGE_XXX macros. */
704#if defined(VBOX_STRICT) && 0 /** @todo triggers in pgmRZDynMapGCPageV2Inlined */
705# define PGM_PAGE_ASSERT_LOCK(a_pVM) PGM_LOCK_ASSERT_OWNER(a_pVM)
706#else
707# define PGM_PAGE_ASSERT_LOCK(a_pVM) do { } while (0)
708#endif
709
710/**
711 * Gets the page state.
712 * @returns page state (PGM_PAGE_STATE_*).
713 * @param a_pPage Pointer to the physical guest page tracking structure.
714 *
715 * @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
716 * builds.
717 */
718#define PGM_PAGE_GET_STATE_NA(a_pPage) ( (a_pPage)->s.uStateY )
719#if defined(__GNUC__) && defined(VBOX_STRICT)
720# define PGM_PAGE_GET_STATE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_STATE_NA(a_pPage); })
721#else
722# define PGM_PAGE_GET_STATE PGM_PAGE_GET_STATE_NA
723#endif
724
725/**
726 * Sets the page state.
727 * @param a_pVM The VM handle, only used for lock ownership assertions.
728 * @param a_pPage Pointer to the physical guest page tracking structure.
729 * @param a_uState The new page state.
730 */
731#define PGM_PAGE_SET_STATE(a_pVM, a_pPage, a_uState) \
732 do { (a_pPage)->s.uStateY = (a_uState); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
733
734
735/**
736 * Gets the host physical address of the guest page.
737 * @returns host physical address (RTHCPHYS).
738 * @param a_pPage Pointer to the physical guest page tracking structure.
739 *
740 * @remarks In strict builds on gcc platforms, this macro will make some ugly
741 * assumption about a valid pVM variable/parameter being in the
742 * current context. It will use this pVM variable to assert that the
743 * PGM lock is held. Use the PGM_PAGE_GET_HCPHYS_NA in contexts where
744 * pVM is not around.
745 */
746#if 0
747# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->s.HCPhysFN << 12 )
748# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA
749#else
750# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->au64[0] & UINT64_C(0x0000fffffffff000) )
751# if defined(__GNUC__) && defined(VBOX_STRICT)
752# define PGM_PAGE_GET_HCPHYS(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_HCPHYS_NA(a_pPage); })
753# else
754# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA
755# endif
756#endif
757
758/**
759 * Sets the host physical address of the guest page.
760 *
761 * @param a_pVM The VM handle, only used for lock ownership assertions.
762 * @param a_pPage Pointer to the physical guest page tracking structure.
763 * @param a_HCPhys The new host physical address.
764 */
765#define PGM_PAGE_SET_HCPHYS(a_pVM, a_pPage, a_HCPhys) \
766 do { \
767 RTHCPHYS const SetHCPhysTmp = (a_HCPhys); \
768 AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
769 (a_pPage)->s.HCPhysFN = SetHCPhysTmp >> 12; \
770 PGM_PAGE_ASSERT_LOCK(a_pVM); \
771 } while (0)
772
773/**
774 * Get the Page ID.
775 * @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
776 * @param a_pPage Pointer to the physical guest page tracking structure.
777 */
778#define PGM_PAGE_GET_PAGEID(a_pPage) ( (uint32_t)(a_pPage)->s.idPage )
779
780/**
781 * Sets the Page ID.
782 * @param a_pVM The VM handle, only used for lock ownership assertions.
783 * @param a_pPage Pointer to the physical guest page tracking structure.
784 * @param a_idPage The new page ID.
785 */
786#define PGM_PAGE_SET_PAGEID(a_pVM, a_pPage, a_idPage) \
787 do { \
788 (a_pPage)->s.idPage = (a_idPage); \
789 PGM_PAGE_ASSERT_LOCK(a_pVM); \
790 } while (0)
791
792/**
793 * Get the Chunk ID.
794 * @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
795 * @param a_pPage Pointer to the physical guest page tracking structure.
796 */
797#define PGM_PAGE_GET_CHUNKID(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) >> GMM_CHUNKID_SHIFT )
798
799/**
800 * Get the index of the page within the allocation chunk.
801 * @returns The page index.
802 * @param a_pPage Pointer to the physical guest page tracking structure.
803 */
804#define PGM_PAGE_GET_PAGE_IN_CHUNK(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) & GMM_PAGEID_IDX_MASK )
805
806/**
807 * Gets the page type.
808 * @returns The page type.
809 * @param a_pPage Pointer to the physical guest page tracking structure.
810 *
811 * @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
812 * builds.
813 */
814#define PGM_PAGE_GET_TYPE_NA(a_pPage) ( (a_pPage)->s.uTypeY )
815#if defined(__GNUC__) && defined(VBOX_STRICT)
816# define PGM_PAGE_GET_TYPE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TYPE_NA(a_pPage); })
817#else
818# define PGM_PAGE_GET_TYPE PGM_PAGE_GET_TYPE_NA
819#endif
820
821/**
822 * Sets the page type.
823 *
824 * @param a_pVM The VM handle, only used for lock ownership assertions.
825 * @param a_pPage Pointer to the physical guest page tracking structure.
826 * @param a_enmType The new page type (PGMPAGETYPE).
827 */
828#define PGM_PAGE_SET_TYPE(a_pVM, a_pPage, a_enmType) \
829 do { (a_pPage)->s.uTypeY = (a_enmType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
830
831/**
832 * Gets the page table index
833 * @returns The page table index.
834 * @param a_pPage Pointer to the physical guest page tracking structure.
835 */
836#define PGM_PAGE_GET_PTE_INDEX(a_pPage) ( (a_pPage)->s.u10PteIdx )
837
838/**
839 * Sets the page table index.
840 * @param a_pVM The VM handle, only used for lock ownership assertions.
841 * @param a_pPage Pointer to the physical guest page tracking structure.
842 * @param a_iPte New page table index.
843 */
844#define PGM_PAGE_SET_PTE_INDEX(a_pVM, a_pPage, a_iPte) \
845 do { (a_pPage)->s.u10PteIdx = (a_iPte); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
846
847/**
848 * Checks if the page is marked for MMIO, no MMIO2 aliasing.
849 * @returns true/false.
850 * @param a_pPage Pointer to the physical guest page tracking structure.
851 */
852#define PGM_PAGE_IS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO )
853
854/**
855 * Checks if the page is marked for MMIO, including both aliases.
856 * @returns true/false.
857 * @param a_pPage Pointer to the physical guest page tracking structure.
858 */
859#define PGM_PAGE_IS_MMIO_OR_ALIAS(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO \
860 || (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO2_ALIAS_MMIO \
861 || (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO \
862 )
863
864/**
865 * Checks if the page is marked for MMIO, including special aliases.
866 * @returns true/false.
867 * @param a_pPage Pointer to the physical guest page tracking structure.
868 */
869#define PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO \
870 || (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO )
871
872/**
873 * Checks if the page is a special aliased MMIO page.
874 * @returns true/false.
875 * @param a_pPage Pointer to the physical guest page tracking structure.
876 */
877#define PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_SPECIAL_ALIAS_MMIO )
878
879/**
880 * Checks if the page is backed by the ZERO page.
881 * @returns true/false.
882 * @param a_pPage Pointer to the physical guest page tracking structure.
883 */
884#define PGM_PAGE_IS_ZERO(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ZERO )
885
886/**
887 * Checks if the page is backed by a SHARED page.
888 * @returns true/false.
889 * @param a_pPage Pointer to the physical guest page tracking structure.
890 */
891#define PGM_PAGE_IS_SHARED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_SHARED )
892
893/**
894 * Checks if the page is ballooned.
895 * @returns true/false.
896 * @param a_pPage Pointer to the physical guest page tracking structure.
897 */
898#define PGM_PAGE_IS_BALLOONED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_BALLOONED )
899
900/**
901 * Checks if the page is allocated.
902 * @returns true/false.
903 * @param a_pPage Pointer to the physical guest page tracking structure.
904 */
905#define PGM_PAGE_IS_ALLOCATED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ALLOCATED )
906
907/**
908 * Marks the page as written to (for GMM change monitoring).
909 * @param a_pVM The VM handle, only used for lock ownership assertions.
910 * @param a_pPage Pointer to the physical guest page tracking structure.
911 */
912#define PGM_PAGE_SET_WRITTEN_TO(a_pVM, a_pPage) \
913 do { (a_pPage)->s.fWrittenToY = 1; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
914
915/**
916 * Clears the written-to indicator.
917 * @param a_pVM The VM handle, only used for lock ownership assertions.
918 * @param a_pPage Pointer to the physical guest page tracking structure.
919 */
920#define PGM_PAGE_CLEAR_WRITTEN_TO(a_pVM, a_pPage) \
921 do { (a_pPage)->s.fWrittenToY = 0; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
922
923/**
924 * Checks if the page was marked as written-to.
925 * @returns true/false.
926 * @param a_pPage Pointer to the physical guest page tracking structure.
927 */
928#define PGM_PAGE_IS_WRITTEN_TO(a_pPage) ( (a_pPage)->s.fWrittenToY )
929
930
931/** @name PT usage values (PGMPAGE::u2PDEType).
932 *
933 * @{ */
934/** Either as a PT or PDE. */
935#define PGM_PAGE_PDE_TYPE_DONTCARE 0
936/** Must use a page table to map the range. */
937#define PGM_PAGE_PDE_TYPE_PT 1
938/** Can use a page directory entry to map the continuous range. */
939#define PGM_PAGE_PDE_TYPE_PDE 2
940/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */
941#define PGM_PAGE_PDE_TYPE_PDE_DISABLED 3
942/** @} */
943
944/**
945 * Set the PDE type of the page
946 * @param a_pVM The VM handle, only used for lock ownership assertions.
947 * @param a_pPage Pointer to the physical guest page tracking structure.
948 * @param a_uType PGM_PAGE_PDE_TYPE_*.
949 */
950#define PGM_PAGE_SET_PDE_TYPE(a_pVM, a_pPage, a_uType) \
951 do { (a_pPage)->s.u2PDETypeY = (a_uType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
952
953/**
954 * Checks if the page was marked being part of a large page
955 * @returns true/false.
956 * @param a_pPage Pointer to the physical guest page tracking structure.
957 */
958#define PGM_PAGE_GET_PDE_TYPE(a_pPage) ( (a_pPage)->s.u2PDETypeY )
959
960/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
961 *
962 * @remarks The values are assigned in order of priority, so we can calculate
963 * the correct state for a page with different handlers installed.
964 * @{ */
965/** No handler installed. */
966#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
967/** Monitoring is temporarily disabled. */
968#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
969/** Write access is monitored. */
970#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
971/** All access is monitored. */
972#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
973/** @} */
974
975/**
976 * Gets the physical access handler state of a page.
977 * @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
978 * @param a_pPage Pointer to the physical guest page tracking structure.
979 */
980#define PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) ( (a_pPage)->s.u2HandlerPhysStateY )
981
982/**
983 * Sets the physical access handler state of a page.
984 * @param a_pPage Pointer to the physical guest page tracking structure.
985 * @param a_uState The new state value.
986 */
987#define PGM_PAGE_SET_HNDL_PHYS_STATE(a_pPage, a_uState) \
988 do { (a_pPage)->s.u2HandlerPhysStateY = (a_uState); } while (0)
989
990/**
991 * Checks if the page has any physical access handlers, including temporarily disabled ones.
992 * @returns true/false
993 * @param a_pPage Pointer to the physical guest page tracking structure.
994 */
995#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(a_pPage) \
996 ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
997
998/**
999 * Checks if the page has any active physical access handlers.
1000 * @returns true/false
1001 * @param a_pPage Pointer to the physical guest page tracking structure.
1002 */
1003#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(a_pPage) \
1004 ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
1005
1006/**
1007 * Checks if the page has any access handlers, including temporarily disabled ones.
1008 * @returns true/false
1009 * @param a_pPage Pointer to the physical guest page tracking structure.
1010 */
1011#define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \
1012 ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
1013
1014/**
1015 * Checks if the page has any active access handlers.
1016 * @returns true/false
1017 * @param a_pPage Pointer to the physical guest page tracking structure.
1018 */
1019#define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \
1020 (PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
1021
1022/**
1023 * Checks if the page has any active access handlers catching all accesses.
1024 * @returns true/false
1025 * @param a_pPage Pointer to the physical guest page tracking structure.
1026 */
1027#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \
1028 ( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL )
1029
1030
1031/** @def PGM_PAGE_GET_TRACKING
1032 * Gets the packed shadow page pool tracking data associated with a guest page.
1033 * @returns uint16_t containing the data.
1034 * @param a_pPage Pointer to the physical guest page tracking structure.
1035 */
1036#define PGM_PAGE_GET_TRACKING_NA(a_pPage) ( (a_pPage)->s.u16TrackingY )
1037#if defined(__GNUC__) && defined(VBOX_STRICT)
1038# define PGM_PAGE_GET_TRACKING(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TRACKING_NA(a_pPage); })
1039#else
1040# define PGM_PAGE_GET_TRACKING PGM_PAGE_GET_TRACKING_NA
1041#endif
1042
1043/** @def PGM_PAGE_SET_TRACKING
1044 * Sets the packed shadow page pool tracking data associated with a guest page.
1045 * @param a_pVM The VM handle, only used for lock ownership assertions.
1046 * @param a_pPage Pointer to the physical guest page tracking structure.
1047 * @param a_u16TrackingData The tracking data to store.
1048 */
1049#define PGM_PAGE_SET_TRACKING(a_pVM, a_pPage, a_u16TrackingData) \
1050 do { (a_pPage)->s.u16TrackingY = (a_u16TrackingData); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
1051
1052/** @def PGM_PAGE_GET_TD_CREFS
1053 * Gets the @a cRefs tracking data member.
1054 * @returns cRefs.
1055 * @param a_pPage Pointer to the physical guest page tracking structure.
1056 */
1057#define PGM_PAGE_GET_TD_CREFS(a_pPage) \
1058 ((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1059#define PGM_PAGE_GET_TD_CREFS_NA(a_pPage) \
1060 ((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1061
1062/** @def PGM_PAGE_GET_TD_IDX
1063 * Gets the @a idx tracking data member.
1064 * @returns idx.
1065 * @param a_pPage Pointer to the physical guest page tracking structure.
1066 */
1067#define PGM_PAGE_GET_TD_IDX(a_pPage) \
1068 ((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1069#define PGM_PAGE_GET_TD_IDX_NA(a_pPage) \
1070 ((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1071
1072
1073/** Max number of locks on a page. */
1074#define PGM_PAGE_MAX_LOCKS UINT8_C(254)
1075
1076/** Get the read lock count.
1077 * @returns count.
1078 * @param a_pPage Pointer to the physical guest page tracking structure.
1079 */
1080#define PGM_PAGE_GET_READ_LOCKS(a_pPage) ( (a_pPage)->s.cReadLocksY )
1081
1082/** Get the write lock count.
1083 * @returns count.
1084 * @param a_pPage Pointer to the physical guest page tracking structure.
1085 */
1086#define PGM_PAGE_GET_WRITE_LOCKS(a_pPage) ( (a_pPage)->s.cWriteLocksY )
1087
1088/** Decrement the read lock counter.
1089 * @param a_pPage Pointer to the physical guest page tracking structure.
1090 */
1091#define PGM_PAGE_DEC_READ_LOCKS(a_pPage) do { --(a_pPage)->s.cReadLocksY; } while (0)
1092
1093/** Decrement the write lock counter.
1094 * @param a_pPage Pointer to the physical guest page tracking structure.
1095 */
1096#define PGM_PAGE_DEC_WRITE_LOCKS(a_pPage) do { --(a_pPage)->s.cWriteLocksY; } while (0)
1097
1098/** Increment the read lock counter.
1099 * @param a_pPage Pointer to the physical guest page tracking structure.
1100 */
1101#define PGM_PAGE_INC_READ_LOCKS(a_pPage) do { ++(a_pPage)->s.cReadLocksY; } while (0)
1102
1103/** Increment the write lock counter.
1104 * @param a_pPage Pointer to the physical guest page tracking structure.
1105 */
1106#define PGM_PAGE_INC_WRITE_LOCKS(a_pPage) do { ++(a_pPage)->s.cWriteLocksY; } while (0)
1107
1108
1109/** Gets the NEM state.
1110 * @returns NEM state value (two bits).
1111 * @param a_pPage Pointer to the physical guest page tracking structure.
1112 */
1113#define PGM_PAGE_GET_NEM_STATE(a_pPage) ((a_pPage)->s.u2NemStateY)
1114
1115/** Sets the NEM state.
1116 * @param a_pPage Pointer to the physical guest page tracking structure.
1117 * @param a_u2State The NEM state value (specific to NEM impl.).
1118 */
1119#define PGM_PAGE_SET_NEM_STATE(a_pPage, a_u2State) \
1120 do { Assert((a_u2State) < 4); (a_pPage)->s.u2NemStateY = (a_u2State); } while (0)
1121
1122
1123#if 0
1124/** Enables sanity checking of write monitoring using CRC-32. */
1125# define PGMLIVESAVERAMPAGE_WITH_CRC32
1126#endif
1127
1128/**
1129 * Per page live save tracking data.
1130 */
1131typedef struct PGMLIVESAVERAMPAGE
1132{
1133 /** Number of times it has been dirtied. */
1134 uint32_t cDirtied : 24;
1135 /** Whether it is currently dirty. */
1136 uint32_t fDirty : 1;
1137 /** Ignore the page.
1138 * This is used for pages that has been MMIO, MMIO2 or ROM pages once. We will
1139 * deal with these after pausing the VM and DevPCI have said it bit about
1140 * remappings. */
1141 uint32_t fIgnore : 1;
1142 /** Was a ZERO page last time around. */
1143 uint32_t fZero : 1;
1144 /** Was a SHARED page last time around. */
1145 uint32_t fShared : 1;
1146 /** Whether the page is/was write monitored in a previous pass. */
1147 uint32_t fWriteMonitored : 1;
1148 /** Whether the page is/was write monitored earlier in this pass. */
1149 uint32_t fWriteMonitoredJustNow : 1;
1150 /** Bits reserved for future use. */
1151 uint32_t u2Reserved : 2;
1152#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1153 /** CRC-32 for the page. This is for internal consistency checks. */
1154 uint32_t u32Crc;
1155#endif
1156} PGMLIVESAVERAMPAGE;
1157#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1158AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
1159#else
1160AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
1161#endif
1162/** Pointer to the per page live save tracking data. */
1163typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
1164
1165/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
1166#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
1167
1168
1169/**
1170 * RAM range for GC Phys to HC Phys conversion.
1171 *
1172 * Can be used for HC Virt to GC Phys and HC Virt to HC Phys
1173 * conversions too, but we'll let MM handle that for now.
1174 *
1175 * This structure is used by linked lists in both GC and HC.
1176 */
1177typedef struct PGMRAMRANGE
1178{
1179 /** Start of the range. Page aligned. */
1180 RTGCPHYS GCPhys;
1181 /** Size of the range. (Page aligned of course). */
1182 RTGCPHYS cb;
1183 /** Pointer to the next RAM range - for R3. */
1184 R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
1185 /** Pointer to the next RAM range - for R0. */
1186 R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
1187 /** PGM_RAM_RANGE_FLAGS_* flags. */
1188 uint32_t fFlags;
1189 /** NEM specific info, UINT32_MAX if not used. */
1190 uint32_t uNemRange;
1191 /** Last address in the range (inclusive). Page aligned (-1). */
1192 RTGCPHYS GCPhysLast;
1193 /** Start of the HC mapping of the range. This is only used for MMIO2 and in NEM mode. */
1194 R3PTRTYPE(void *) pvR3;
1195 /** Live save per page tracking data. */
1196 R3PTRTYPE(PPGMLIVESAVERAMPAGE) paLSPages;
1197 /** The range description. */
1198 R3PTRTYPE(const char *) pszDesc;
1199 /** Pointer to self - R0 pointer. */
1200 R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1201
1202 /** Pointer to the left search three node - ring-3 context. */
1203 R3PTRTYPE(struct PGMRAMRANGE *) pLeftR3;
1204 /** Pointer to the right search three node - ring-3 context. */
1205 R3PTRTYPE(struct PGMRAMRANGE *) pRightR3;
1206 /** Pointer to the left search three node - ring-0 context. */
1207 R0PTRTYPE(struct PGMRAMRANGE *) pLeftR0;
1208 /** Pointer to the right search three node - ring-0 context. */
1209 R0PTRTYPE(struct PGMRAMRANGE *) pRightR0;
1210
1211 /** Padding to make aPage aligned on sizeof(PGMPAGE). */
1212#if HC_ARCH_BITS == 32
1213 uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 0];
1214#endif
1215 /** Array of physical guest page tracking structures. */
1216 PGMPAGE aPages[1];
1217} PGMRAMRANGE;
1218/** Pointer to RAM range for GC Phys to HC Phys conversion. */
1219typedef PGMRAMRANGE *PPGMRAMRANGE;
1220
1221/** @name PGMRAMRANGE::fFlags
1222 * @{ */
1223/** The RAM range is floating around as an independent guest mapping. */
1224#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1225/** Ad hoc RAM range for an ROM mapping. */
1226#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM RT_BIT(21)
1227/** Ad hoc RAM range for an MMIO mapping. */
1228#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO RT_BIT(22)
1229/** Ad hoc RAM range for an MMIO2 or pre-registered MMIO mapping. */
1230#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX RT_BIT(23)
1231/** @} */
1232
1233/** Tests if a RAM range is an ad hoc one or not.
1234 * @returns true/false.
1235 * @param pRam The RAM range.
1236 */
1237#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
1238 (!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO | PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX) ) )
1239
1240/** The number of entries in the RAM range TLBs (there is one for each
1241 * context). Must be a power of two. */
1242#define PGM_RAMRANGE_TLB_ENTRIES 8
1243
1244/**
1245 * Calculates the RAM range TLB index for the physical address.
1246 *
1247 * @returns RAM range TLB index.
1248 * @param a_GCPhys The guest physical address.
1249 */
1250#define PGM_RAMRANGE_TLB_IDX(a_GCPhys) ( ((a_GCPhys) >> 20) & (PGM_RAMRANGE_TLB_ENTRIES - 1) )
1251
1252/**
1253 * Calculates the ring-3 address for a_GCPhysPage if the RAM range has a
1254 * mapping address.
1255 */
1256#define PGM_RAMRANGE_CALC_PAGE_R3PTR(a_pRam, a_GCPhysPage) \
1257 ( (a_pRam)->pvR3 ? (R3PTRTYPE(uint8_t *))(a_pRam)->pvR3 + (a_GCPhysPage) - (a_pRam)->GCPhys : NULL )
1258
1259
1260/**
1261 * Per page tracking structure for ROM image.
1262 *
1263 * A ROM image may have a shadow page, in which case we may have two pages
1264 * backing it. This structure contains the PGMPAGE for both while
1265 * PGMRAMRANGE have a copy of the active one. It is important that these
1266 * aren't out of sync in any regard other than page pool tracking data.
1267 */
1268typedef struct PGMROMPAGE
1269{
1270 /** The page structure for the virgin ROM page. */
1271 PGMPAGE Virgin;
1272 /** The page structure for the shadow RAM page. */
1273 PGMPAGE Shadow;
1274 /** The current protection setting. */
1275 PGMROMPROT enmProt;
1276 /** Live save status information. Makes use of unused alignment space. */
1277 struct
1278 {
1279 /** The previous protection value. */
1280 uint8_t u8Prot;
1281 /** Written to flag set by the handler. */
1282 bool fWrittenTo;
1283 /** Whether the shadow page is dirty or not. */
1284 bool fDirty;
1285 /** Whether it was dirtied in the recently. */
1286 bool fDirtiedRecently;
1287 } LiveSave;
1288} PGMROMPAGE;
1289AssertCompileSizeAlignment(PGMROMPAGE, 8);
1290/** Pointer to a ROM page tracking structure. */
1291typedef PGMROMPAGE *PPGMROMPAGE;
1292
1293
1294/**
1295 * A registered ROM image.
1296 *
1297 * This is needed to keep track of ROM image since they generally intrude
1298 * into a PGMRAMRANGE. It also keeps track of additional info like the
1299 * two page sets (read-only virgin and read-write shadow), the current
1300 * state of each page.
1301 *
1302 * Because access handlers cannot easily be executed in a different
1303 * context, the ROM ranges needs to be accessible and in all contexts.
1304 */
1305typedef struct PGMROMRANGE
1306{
1307 /** Pointer to the next range - R3. */
1308 R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1309 /** Pointer to the next range - R0. */
1310 R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1311 /** Address of the range. */
1312 RTGCPHYS GCPhys;
1313 /** Address of the last byte in the range. */
1314 RTGCPHYS GCPhysLast;
1315 /** Size of the range. */
1316 RTGCPHYS cb;
1317 /** The flags (PGMPHYS_ROM_FLAGS_*). */
1318 uint8_t fFlags;
1319 /** The saved state range ID. */
1320 uint8_t idSavedState;
1321 /** Alignment padding. */
1322 uint8_t au8Alignment[2];
1323 /** The size bits pvOriginal points to. */
1324 uint32_t cbOriginal;
1325 /** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1326 * This is used for strictness checks. */
1327 R3PTRTYPE(const void *) pvOriginal;
1328 /** The ROM description. */
1329 R3PTRTYPE(const char *) pszDesc;
1330#ifdef VBOX_WITH_PGM_NEM_MODE
1331 /** In simplified memory mode this provides alternate backing for shadowed ROMs.
1332 * - PGMROMPROT_READ_ROM_WRITE_IGNORE: Shadow
1333 * - PGMROMPROT_READ_ROM_WRITE_RAM: Shadow
1334 * - PGMROMPROT_READ_RAM_WRITE_IGNORE: ROM
1335 * - PGMROMPROT_READ_RAM_WRITE_RAM: ROM */
1336 R3PTRTYPE(uint8_t *) pbR3Alternate;
1337 RTR3PTR pvAlignment2;
1338#endif
1339 /** The per page tracking structures. */
1340 PGMROMPAGE aPages[1];
1341} PGMROMRANGE;
1342/** Pointer to a ROM range. */
1343typedef PGMROMRANGE *PPGMROMRANGE;
1344
1345
1346/**
1347 * Live save per page data for an MMIO2 page.
1348 *
1349 * Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
1350 * of MMIO2 pages. The current approach is using some optimistic SHA-1 +
1351 * CRC-32 for detecting changes as well as special handling of zero pages. This
1352 * is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
1353 * for speeding things up. (We're using SHA-1 and not SHA-256 or SHA-512
1354 * because of speed (2.5x and 6x slower).)
1355 *
1356 * @todo Implement dirty MMIO2 page reporting that can be enabled during live
1357 * save but normally is disabled. Since we can write monitor guest
1358 * accesses on our own, we only need this for host accesses. Shouldn't be
1359 * too difficult for DevVGA, VMMDev might be doable, the planned
1360 * networking fun will be fun since it involves ring-0.
1361 */
1362typedef struct PGMLIVESAVEMMIO2PAGE
1363{
1364 /** Set if the page is considered dirty. */
1365 bool fDirty;
1366 /** The number of scans this page has remained unchanged for.
1367 * Only updated for dirty pages. */
1368 uint8_t cUnchangedScans;
1369 /** Whether this page was zero at the last scan. */
1370 bool fZero;
1371 /** Alignment padding. */
1372 bool fReserved;
1373 /** CRC-32 for the first half of the page.
1374 * This is used together with u32CrcH2 to quickly detect changes in the page
1375 * during the non-final passes. */
1376 uint32_t u32CrcH1;
1377 /** CRC-32 for the second half of the page. */
1378 uint32_t u32CrcH2;
1379 /** SHA-1 for the saved page.
1380 * This is used in the final pass to skip pages without changes. */
1381 uint8_t abSha1Saved[RTSHA1_HASH_SIZE];
1382} PGMLIVESAVEMMIO2PAGE;
1383/** Pointer to a live save status data for an MMIO2 page. */
1384typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
1385
1386/**
1387 * A registered MMIO2 (= Device RAM) range.
1388 *
1389 * There are a few reason why we need to keep track of these registrations. One
1390 * of them is the deregistration & cleanup stuff, while another is that the
1391 * PGMRAMRANGE associated with such a region may have to be removed from the ram
1392 * range list.
1393 *
1394 * Overlapping with a RAM range has to be 100% or none at all. The pages in the
1395 * existing RAM range must not be ROM nor MMIO. A guru meditation will be
1396 * raised if a partial overlap or an overlap of ROM pages is encountered. On an
1397 * overlap we will free all the existing RAM pages and put in the ram range
1398 * pages instead.
1399 */
1400typedef struct PGMREGMMIO2RANGE
1401{
1402 /** The owner of the range. (a device) */
1403 PPDMDEVINSR3 pDevInsR3;
1404 /** Pointer to the ring-3 mapping of the allocation. */
1405 RTR3PTR pvR3;
1406#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1407 /** Pointer to the ring-0 mapping of the allocation. */
1408 RTR0PTR pvR0;
1409#endif
1410 /** Pointer to the next range - R3. */
1411 R3PTRTYPE(struct PGMREGMMIO2RANGE *) pNextR3;
1412 /** Flags (PGMREGMMIO2RANGE_F_XXX). */
1413 uint16_t fFlags;
1414 /** The sub device number (internal PCI config (CFGM) number). */
1415 uint8_t iSubDev;
1416 /** The PCI region number. */
1417 uint8_t iRegion;
1418 /** The saved state range ID. */
1419 uint8_t idSavedState;
1420 /** MMIO2 range identifier, for page IDs (PGMPAGE::s.idPage). */
1421 uint8_t idMmio2;
1422 /** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */
1423#ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1424 uint8_t abAlignment[HC_ARCH_BITS == 32 ? 6 + 4 : 2];
1425#else
1426 uint8_t abAlignment[HC_ARCH_BITS == 32 ? 6 + 8 : 2 + 8];
1427#endif
1428 /** The real size.
1429 * This may be larger than indicated by RamRange.cb if the range has been
1430 * reduced during saved state loading. */
1431 RTGCPHYS cbReal;
1432 /** Pointer to the physical handler for MMIO.
1433 * If NEM is responsible for tracking dirty pages in simple memory mode, this
1434 * will be NULL. */
1435 R3PTRTYPE(PPGMPHYSHANDLER) pPhysHandlerR3;
1436 /** Live save per page tracking data for MMIO2. */
1437 R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE) paLSPages;
1438 /** The associated RAM range. */
1439 PGMRAMRANGE RamRange;
1440} PGMREGMMIO2RANGE;
1441AssertCompileMemberAlignment(PGMREGMMIO2RANGE, RamRange, 16);
1442/** Pointer to a MMIO2 or pre-registered MMIO range. */
1443typedef PGMREGMMIO2RANGE *PPGMREGMMIO2RANGE;
1444
1445/** @name PGMREGMMIO2RANGE_F_XXX - Registered MMIO2 range flags.
1446 * @{ */
1447/** Set if this is the first chunk in the MMIO2 range. */
1448#define PGMREGMMIO2RANGE_F_FIRST_CHUNK UINT16_C(0x0001)
1449/** Set if this is the last chunk in the MMIO2 range. */
1450#define PGMREGMMIO2RANGE_F_LAST_CHUNK UINT16_C(0x0002)
1451/** Set if the whole range is mapped. */
1452#define PGMREGMMIO2RANGE_F_MAPPED UINT16_C(0x0004)
1453/** Set if it's overlapping, clear if not. */
1454#define PGMREGMMIO2RANGE_F_OVERLAPPING UINT16_C(0x0008)
1455/** This mirrors the PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES creation flag.*/
1456#define PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES UINT16_C(0x0010)
1457/** Set if the access handler is registered. */
1458#define PGMREGMMIO2RANGE_F_IS_TRACKING UINT16_C(0x0020)
1459/** Set if dirty page tracking is currently enabled. */
1460#define PGMREGMMIO2RANGE_F_TRACKING_ENABLED UINT16_C(0x0040)
1461/** Set if there are dirty pages in the range. */
1462#define PGMREGMMIO2RANGE_F_IS_DIRTY UINT16_C(0x0080)
1463/** @} */
1464
1465
1466/** @name Internal MMIO2 constants.
1467 * @{ */
1468/** The maximum number of MMIO2 ranges. */
1469#define PGM_MMIO2_MAX_RANGES 32
1470/** The maximum number of pages in a MMIO2 range. */
1471#define PGM_MMIO2_MAX_PAGE_COUNT UINT32_C(0x01000000)
1472/** Makes a MMIO2 page ID out of a MMIO2 range ID and page index number. */
1473#define PGM_MMIO2_PAGEID_MAKE(a_idMmio2, a_iPage) ( ((uint32_t)(a_idMmio2) << 24) | (uint32_t)(a_iPage) )
1474/** Gets the MMIO2 range ID from an MMIO2 page ID. */
1475#define PGM_MMIO2_PAGEID_GET_MMIO2_ID(a_idPage) ( (uint8_t)((a_idPage) >> 24) )
1476/** Gets the MMIO2 page index from an MMIO2 page ID. */
1477#define PGM_MMIO2_PAGEID_GET_IDX(a_idPage) ( ((a_idPage) & UINT32_C(0x00ffffff)) )
1478/** @} */
1479
1480
1481
1482/**
1483 * PGMPhysRead/Write cache entry
1484 */
1485typedef struct PGMPHYSCACHEENTRY
1486{
1487 /** R3 pointer to physical page. */
1488 R3PTRTYPE(uint8_t *) pbR3;
1489 /** GC Physical address for cache entry */
1490 RTGCPHYS GCPhys;
1491#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1492 RTGCPHYS u32Padding0; /**< alignment padding. */
1493#endif
1494} PGMPHYSCACHEENTRY;
1495
1496/**
1497 * PGMPhysRead/Write cache to reduce REM memory access overhead
1498 */
1499typedef struct PGMPHYSCACHE
1500{
1501 /** Bitmap of valid cache entries */
1502 uint64_t aEntries;
1503 /** Cache entries */
1504 PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1505} PGMPHYSCACHE;
1506
1507
1508/** @name Ring-3 page mapping TLBs
1509 * @{ */
1510
1511/** Pointer to an allocation chunk ring-3 mapping. */
1512typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1513/** Pointer to an allocation chunk ring-3 mapping pointer. */
1514typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1515
1516/**
1517 * Ring-3 tracking structure for an allocation chunk ring-3 mapping.
1518 *
1519 * The primary tree (Core) uses the chunk id as key.
1520 */
1521typedef struct PGMCHUNKR3MAP
1522{
1523 /** The key is the chunk id. */
1524 AVLU32NODECORE Core;
1525 /** The time (ChunkR3Map.iNow) this chunk was last used. Used for unmap
1526 * selection. */
1527 uint32_t iLastUsed;
1528 /** The current reference count. */
1529 uint32_t volatile cRefs;
1530 /** The current permanent reference count. */
1531 uint32_t volatile cPermRefs;
1532 /** The mapping address. */
1533 void *pv;
1534} PGMCHUNKR3MAP;
1535
1536/**
1537 * Allocation chunk ring-3 mapping TLB entry.
1538 */
1539typedef struct PGMCHUNKR3MAPTLBE
1540{
1541 /** The chunk id. */
1542 uint32_t volatile idChunk;
1543#if HC_ARCH_BITS == 64
1544 uint32_t u32Padding; /**< alignment padding. */
1545#endif
1546 /** The chunk map. */
1547 R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1548} PGMCHUNKR3MAPTLBE;
1549/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1550typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1551
1552/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1553 * @remark Must be a power of two value. */
1554#define PGM_CHUNKR3MAPTLB_ENTRIES 64
1555
1556/**
1557 * Allocation chunk ring-3 mapping TLB.
1558 *
1559 * @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1560 * At first glance this might look kinda odd since AVL trees are
1561 * supposed to give the most optimal lookup times of all trees
1562 * due to their balancing. However, take a tree with 1023 nodes
1563 * in it, that's 10 levels, meaning that most searches has to go
1564 * down 9 levels before they find what they want. This isn't fast
1565 * compared to a TLB hit. There is the factor of cache misses,
1566 * and of course the problem with trees and branch prediction.
1567 * This is why we use TLBs in front of most of the trees.
1568 *
1569 * @todo Generalize this TLB + AVL stuff, shouldn't be all that
1570 * difficult when we switch to the new inlined AVL trees (from kStuff).
1571 */
1572typedef struct PGMCHUNKR3MAPTLB
1573{
1574 /** The TLB entries. */
1575 PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1576} PGMCHUNKR3MAPTLB;
1577
1578/**
1579 * Calculates the index of a guest page in the Ring-3 Chunk TLB.
1580 * @returns Chunk TLB index.
1581 * @param idChunk The Chunk ID.
1582 */
1583#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1584
1585
1586/**
1587 * Ring-3 guest page mapping TLB entry.
1588 * @remarks used in ring-0 as well at the moment.
1589 */
1590typedef struct PGMPAGER3MAPTLBE
1591{
1592 /** Address of the page. */
1593 RTGCPHYS volatile GCPhys;
1594 /** The guest page. */
1595 R3PTRTYPE(PPGMPAGE) volatile pPage;
1596 /** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1597 R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1598 /** The address */
1599 R3PTRTYPE(void *) volatile pv;
1600#if HC_ARCH_BITS == 32
1601 uint32_t u32Padding; /**< alignment padding. */
1602#endif
1603} PGMPAGER3MAPTLBE;
1604/** Pointer to an entry in the HC physical TLB. */
1605typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1606
1607
1608/** The number of entries in the ring-3 guest page mapping TLB.
1609 * @remarks The value must be a power of two. */
1610#define PGM_PAGER3MAPTLB_ENTRIES 256
1611
1612/**
1613 * Ring-3 guest page mapping TLB.
1614 * @remarks used in ring-0 as well at the moment.
1615 */
1616typedef struct PGMPAGER3MAPTLB
1617{
1618 /** The TLB entries. */
1619 PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1620} PGMPAGER3MAPTLB;
1621/** Pointer to the ring-3 guest page mapping TLB. */
1622typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1623
1624/**
1625 * Calculates the index of the TLB entry for the specified guest page.
1626 * @returns Physical TLB index.
1627 * @param GCPhys The guest physical address.
1628 */
1629#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1630
1631/** @} */
1632
1633
1634/** @name Ring-0 page mapping TLB
1635 * @{ */
1636/**
1637 * Ring-0 guest page mapping TLB entry.
1638 */
1639typedef struct PGMPAGER0MAPTLBE
1640{
1641 /** Address of the page. */
1642 RTGCPHYS volatile GCPhys;
1643 /** The guest page. */
1644 R0PTRTYPE(PPGMPAGE) volatile pPage;
1645 /** The address */
1646 R0PTRTYPE(void *) volatile pv;
1647} PGMPAGER0MAPTLBE;
1648/** Pointer to an entry in the HC physical TLB. */
1649typedef PGMPAGER0MAPTLBE *PPGMPAGER0MAPTLBE;
1650
1651
1652/** The number of entries in the ring-3 guest page mapping TLB.
1653 * @remarks The value must be a power of two. */
1654#define PGM_PAGER0MAPTLB_ENTRIES 256
1655
1656/**
1657 * Ring-3 guest page mapping TLB.
1658 * @remarks used in ring-0 as well at the moment.
1659 */
1660typedef struct PGMPAGER0MAPTLB
1661{
1662 /** The TLB entries. */
1663 PGMPAGER0MAPTLBE aEntries[PGM_PAGER0MAPTLB_ENTRIES];
1664} PGMPAGER0MAPTLB;
1665/** Pointer to the ring-3 guest page mapping TLB. */
1666typedef PGMPAGER0MAPTLB *PPGMPAGER0MAPTLB;
1667
1668/**
1669 * Calculates the index of the TLB entry for the specified guest page.
1670 * @returns Physical TLB index.
1671 * @param GCPhys The guest physical address.
1672 */
1673#define PGM_PAGER0MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER0MAPTLB_ENTRIES - 1) )
1674/** @} */
1675
1676
1677/** @name Context neutral page mapper TLB.
1678 *
1679 * Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1680 * code is writting in a kind of context neutral way. Time will show whether
1681 * this actually makes sense or not...
1682 *
1683 * @todo this needs to be reconsidered and dropped/redone since the ring-0
1684 * context ends up using a global mapping cache on some platforms
1685 * (darwin).
1686 *
1687 * @{ */
1688/** @typedef PPGMPAGEMAPTLB
1689 * The page mapper TLB pointer type for the current context. */
1690/** @typedef PPGMPAGEMAPTLB
1691 * The page mapper TLB entry pointer type for the current context. */
1692/** @typedef PPGMPAGEMAPTLB
1693 * The page mapper TLB entry pointer pointer type for the current context. */
1694/** @def PGM_PAGEMAPTLB_ENTRIES
1695 * The number of TLB entries in the page mapper TLB for the current context. */
1696/** @def PGM_PAGEMAPTLB_IDX
1697 * Calculate the TLB index for a guest physical address.
1698 * @returns The TLB index.
1699 * @param GCPhys The guest physical address. */
1700/** @typedef PPGMPAGEMAP
1701 * Pointer to a page mapper unit for current context. */
1702/** @typedef PPPGMPAGEMAP
1703 * Pointer to a page mapper unit pointer for current context. */
1704#if defined(IN_RING0)
1705typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1706typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1707typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1708# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1709# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1710typedef struct PGMCHUNKR0MAP *PPGMPAGEMAP;
1711typedef struct PGMCHUNKR0MAP **PPPGMPAGEMAP;
1712#else
1713typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1714typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1715typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1716# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1717# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1718typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1719typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1720#endif
1721/** @} */
1722
1723
1724/** @name PGM Pool Indexes.
1725 * Aka. the unique shadow page identifier.
1726 * @{ */
1727/** NIL page pool IDX. */
1728#define NIL_PGMPOOL_IDX 0
1729/** The first normal index. There used to be 5 fictive pages up front, now
1730 * there is only the NIL page. */
1731#define PGMPOOL_IDX_FIRST 1
1732/** The last valid index. (inclusive, 14 bits) */
1733#define PGMPOOL_IDX_LAST 0x3fff
1734/** @} */
1735
1736/** The NIL index for the parent chain. */
1737#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1738#define NIL_PGMPOOL_PRESENT_INDEX ((uint16_t)0xffff)
1739
1740/**
1741 * Node in the chain linking a shadowed page to it's parent (user).
1742 */
1743#pragma pack(1)
1744typedef struct PGMPOOLUSER
1745{
1746 /** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1747 uint16_t iNext;
1748 /** The user page index. */
1749 uint16_t iUser;
1750 /** Index into the user table. */
1751 uint32_t iUserTable;
1752} PGMPOOLUSER, *PPGMPOOLUSER;
1753typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1754#pragma pack()
1755
1756
1757/** The NIL index for the phys ext chain. */
1758#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1759/** The NIL pte index for a phys ext chain slot. */
1760#define NIL_PGMPOOL_PHYSEXT_IDX_PTE ((uint16_t)0xffff)
1761
1762/**
1763 * Node in the chain of physical cross reference extents.
1764 * @todo Calling this an 'extent' is not quite right, find a better name.
1765 * @todo find out the optimal size of the aidx array
1766 */
1767#pragma pack(1)
1768typedef struct PGMPOOLPHYSEXT
1769{
1770 /** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1771 uint16_t iNext;
1772 /** Alignment. */
1773 uint16_t u16Align;
1774 /** The user page index. */
1775 uint16_t aidx[3];
1776 /** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */
1777 uint16_t apte[3];
1778} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1779typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1780#pragma pack()
1781
1782
1783/**
1784 * The kind of page that's being shadowed.
1785 */
1786typedef enum PGMPOOLKIND
1787{
1788 /** The virtual invalid 0 entry. */
1789 PGMPOOLKIND_INVALID = 0,
1790 /** The entry is free (=unused). */
1791 PGMPOOLKIND_FREE,
1792
1793 /** Shw: 32-bit page table; Gst: no paging. */
1794 PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1795 /** Shw: 32-bit page table; Gst: 32-bit page table. */
1796 PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1797 /** Shw: 32-bit page table; Gst: 4MB page. */
1798 PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1799 /** Shw: PAE page table; Gst: no paging. */
1800 PGMPOOLKIND_PAE_PT_FOR_PHYS,
1801 /** Shw: PAE page table; Gst: 32-bit page table. */
1802 PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1803 /** Shw: PAE page table; Gst: Half of a 4MB page. */
1804 PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1805 /** Shw: PAE page table; Gst: PAE page table. */
1806 PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1807 /** Shw: PAE page table; Gst: 2MB page. */
1808 PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1809
1810 /** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1811 PGMPOOLKIND_32BIT_PD,
1812 /** Shw: 32-bit page directory. Gst: no paging. */
1813 PGMPOOLKIND_32BIT_PD_PHYS,
1814 /** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1815 PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1816 /** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1817 PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1818 /** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1819 PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1820 /** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1821 PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1822 /** Shw: PAE page directory; Gst: PAE page directory. */
1823 PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1824 /** Shw: PAE page directory; Gst: no paging. Note: +NP. */
1825 PGMPOOLKIND_PAE_PD_PHYS,
1826
1827 /** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1828 PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1829 /** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1830 PGMPOOLKIND_PAE_PDPT,
1831 /** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1832 PGMPOOLKIND_PAE_PDPT_PHYS,
1833
1834 /** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1835 PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1836 /** Shw: 64-bit page directory pointer table; Gst: no paging. */
1837 PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1838 /** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1839 PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1840 /** Shw: 64-bit page directory table; Gst: no paging. */
1841 PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 24 */
1842
1843 /** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1844 PGMPOOLKIND_64BIT_PML4,
1845
1846 /** Shw: EPT page directory pointer table; Gst: no paging. */
1847 PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1848 /** Shw: EPT page directory table; Gst: no paging. */
1849 PGMPOOLKIND_EPT_PD_FOR_PHYS,
1850 /** Shw: EPT page table; Gst: no paging. */
1851 PGMPOOLKIND_EPT_PT_FOR_PHYS,
1852
1853 /** Shw: Root Nested paging table. */
1854 PGMPOOLKIND_ROOT_NESTED,
1855
1856 /** The last valid entry. */
1857 PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1858} PGMPOOLKIND;
1859
1860/**
1861 * The access attributes of the page; only applies to big pages.
1862 */
1863typedef enum
1864{
1865 PGMPOOLACCESS_DONTCARE = 0,
1866 PGMPOOLACCESS_USER_RW,
1867 PGMPOOLACCESS_USER_R,
1868 PGMPOOLACCESS_USER_RW_NX,
1869 PGMPOOLACCESS_USER_R_NX,
1870 PGMPOOLACCESS_SUPERVISOR_RW,
1871 PGMPOOLACCESS_SUPERVISOR_R,
1872 PGMPOOLACCESS_SUPERVISOR_RW_NX,
1873 PGMPOOLACCESS_SUPERVISOR_R_NX
1874} PGMPOOLACCESS;
1875
1876/**
1877 * The tracking data for a page in the pool.
1878 */
1879typedef struct PGMPOOLPAGE
1880{
1881 /** AVL node code with the (HC) physical address of this page. */
1882 AVLOHCPHYSNODECORE Core;
1883 /** Pointer to the R3 mapping of the page. */
1884 R3PTRTYPE(void *) pvPageR3;
1885 /** Pointer to the R0 mapping of the page. */
1886 R0PTRTYPE(void *) pvPageR0;
1887 /** The guest physical address. */
1888 RTGCPHYS GCPhys;
1889 /** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1890 uint8_t enmKind;
1891 /** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
1892 uint8_t enmAccess;
1893 /** This supplements enmKind and enmAccess */
1894 bool fA20Enabled : 1;
1895
1896 /** Used to indicate that the page is zeroed. */
1897 bool fZeroed : 1;
1898 /** Used to indicate that a PT has non-global entries. */
1899 bool fSeenNonGlobal : 1;
1900 /** Used to indicate that we're monitoring writes to the guest page. */
1901 bool fMonitored : 1;
1902 /** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1903 * (All pages are in the age list.) */
1904 bool fCached : 1;
1905 /** This is used by the R3 access handlers when invoked by an async thread.
1906 * It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1907 bool volatile fReusedFlushPending : 1;
1908 /** Used to mark the page as dirty (write monitoring is temporarily
1909 * off). */
1910 bool fDirty : 1;
1911 bool fPadding1 : 1;
1912 bool fPadding2;
1913
1914 /** The index of this page. */
1915 uint16_t idx;
1916 /** The next entry in the list this page currently resides in.
1917 * It's either in the free list or in the GCPhys hash. */
1918 uint16_t iNext;
1919 /** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1920 uint16_t iUserHead;
1921 /** The number of present entries. */
1922 uint16_t cPresent;
1923 /** The first entry in the table which is present. */
1924 uint16_t iFirstPresent;
1925 /** The number of modifications to the monitored page. */
1926 uint16_t cModifications;
1927 /** The next modified page. NIL_PGMPOOL_IDX if tail. */
1928 uint16_t iModifiedNext;
1929 /** The previous modified page. NIL_PGMPOOL_IDX if head. */
1930 uint16_t iModifiedPrev;
1931 /** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1932 uint16_t iMonitoredNext;
1933 /** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1934 uint16_t iMonitoredPrev;
1935 /** The next page in the age list. */
1936 uint16_t iAgeNext;
1937 /** The previous page in the age list. */
1938 uint16_t iAgePrev;
1939 /** Index into PGMPOOL::aDirtyPages if fDirty is set. */
1940 uint8_t idxDirtyEntry;
1941
1942 /** @name Access handler statistics to determine whether the guest is
1943 * (re)initializing a page table.
1944 * @{ */
1945 RTGCPTR GCPtrLastAccessHandlerRip;
1946 RTGCPTR GCPtrLastAccessHandlerFault;
1947 uint64_t cLastAccessHandler;
1948 /** @} */
1949 /** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages. */
1950 uint32_t volatile cLocked;
1951#if GC_ARCH_BITS == 64
1952 uint32_t u32Alignment3;
1953#endif
1954# ifdef VBOX_STRICT
1955 RTGCPTR GCPtrDirtyFault;
1956# endif
1957} PGMPOOLPAGE;
1958/** Pointer to a pool page. */
1959typedef PGMPOOLPAGE *PPGMPOOLPAGE;
1960/** Pointer to a const pool page. */
1961typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1962/** Pointer to a pool page pointer. */
1963typedef PGMPOOLPAGE **PPPGMPOOLPAGE;
1964
1965
1966/** The hash table size. */
1967# define PGMPOOL_HASH_SIZE 0x40
1968/** The hash function. */
1969# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1970
1971
1972/**
1973 * The shadow page pool instance data.
1974 *
1975 * It's all one big allocation made at init time, except for the
1976 * pages that is. The user nodes follows immediately after the
1977 * page structures.
1978 */
1979typedef struct PGMPOOL
1980{
1981 /** The VM handle - R3 Ptr. */
1982 PVMR3 pVMR3;
1983 /** The VM handle - R0 Ptr. */
1984 R0PTRTYPE(PVMCC) pVMR0;
1985 /** The max pool size. This includes the special IDs. */
1986 uint16_t cMaxPages;
1987 /** The current pool size. */
1988 uint16_t cCurPages;
1989 /** The head of the free page list. */
1990 uint16_t iFreeHead;
1991 /* Padding. */
1992 uint16_t u16Padding;
1993 /** Head of the chain of free user nodes. */
1994 uint16_t iUserFreeHead;
1995 /** The number of user nodes we've allocated. */
1996 uint16_t cMaxUsers;
1997 /** The number of present page table entries in the entire pool. */
1998 uint32_t cPresent;
1999 /** Pointer to the array of user nodes - R3 pointer. */
2000 R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
2001 /** Pointer to the array of user nodes - R0 pointer. */
2002 R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
2003 /** Head of the chain of free phys ext nodes. */
2004 uint16_t iPhysExtFreeHead;
2005 /** The number of user nodes we've allocated. */
2006 uint16_t cMaxPhysExts;
2007 uint32_t u32Padding0b;
2008 /** Pointer to the array of physical xref extent nodes - R3 pointer. */
2009 R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
2010 /** Pointer to the array of physical xref extent nodes - R0 pointer. */
2011 R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
2012 /** Hash table for GCPhys addresses. */
2013 uint16_t aiHash[PGMPOOL_HASH_SIZE];
2014 /** The head of the age list. */
2015 uint16_t iAgeHead;
2016 /** The tail of the age list. */
2017 uint16_t iAgeTail;
2018 /** Set if the cache is enabled. */
2019 bool fCacheEnabled;
2020 /** Alignment padding. */
2021 bool afPadding1[3];
2022 /** Head of the list of modified pages. */
2023 uint16_t iModifiedHead;
2024 /** The current number of modified pages. */
2025 uint16_t cModifiedPages;
2026 /** Physical access handler type registration handle. */
2027 PGMPHYSHANDLERTYPE hAccessHandlerType;
2028 /** Next available slot (in aDirtyPages). */
2029 uint32_t idxFreeDirtyPage;
2030 /** Number of active dirty pages. */
2031 uint32_t cDirtyPages;
2032 /** Array of current dirty pgm pool page indices. */
2033 uint16_t aidxDirtyPages[16];
2034 /** Array running in parallel to aidxDirtyPages with the page data. */
2035 struct
2036 {
2037 uint64_t aPage[512];
2038 } aDirtyPages[16];
2039
2040 /** The number of pages currently in use. */
2041 uint16_t cUsedPages;
2042#ifdef VBOX_WITH_STATISTICS
2043 /** The high water mark for cUsedPages. */
2044 uint16_t cUsedPagesHigh;
2045 uint32_t Alignment1; /**< Align the next member on a 64-bit boundary. */
2046 /** Profiling pgmPoolAlloc(). */
2047 STAMPROFILEADV StatAlloc;
2048 /** Profiling pgmR3PoolClearDoIt(). */
2049 STAMPROFILE StatClearAll;
2050 /** Profiling pgmR3PoolReset(). */
2051 STAMPROFILE StatR3Reset;
2052 /** Profiling pgmPoolFlushPage(). */
2053 STAMPROFILE StatFlushPage;
2054 /** Profiling pgmPoolFree(). */
2055 STAMPROFILE StatFree;
2056 /** Counting explicit flushes by PGMPoolFlushPage(). */
2057 STAMCOUNTER StatForceFlushPage;
2058 /** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
2059 STAMCOUNTER StatForceFlushDirtyPage;
2060 /** Counting flushes for reused pages. */
2061 STAMCOUNTER StatForceFlushReused;
2062 /** Profiling time spent zeroing pages. */
2063 STAMPROFILE StatZeroPage;
2064 /** Profiling of pgmPoolTrackDeref. */
2065 STAMPROFILE StatTrackDeref;
2066 /** Profiling pgmTrackFlushGCPhysPT. */
2067 STAMPROFILE StatTrackFlushGCPhysPT;
2068 /** Profiling pgmTrackFlushGCPhysPTs. */
2069 STAMPROFILE StatTrackFlushGCPhysPTs;
2070 /** Profiling pgmTrackFlushGCPhysPTsSlow. */
2071 STAMPROFILE StatTrackFlushGCPhysPTsSlow;
2072 /** Number of times we've been out of user records. */
2073 STAMCOUNTER StatTrackFreeUpOneUser;
2074 /** Nr of flushed entries. */
2075 STAMCOUNTER StatTrackFlushEntry;
2076 /** Nr of updated entries. */
2077 STAMCOUNTER StatTrackFlushEntryKeep;
2078 /** Profiling deref activity related tracking GC physical pages. */
2079 STAMPROFILE StatTrackDerefGCPhys;
2080 /** Number of linear searches for a HCPhys in the ram ranges. */
2081 STAMCOUNTER StatTrackLinearRamSearches;
2082 /** The number of failing pgmPoolTrackPhysExtAlloc calls. */
2083 STAMCOUNTER StamTrackPhysExtAllocFailures;
2084
2085 /** Profiling the RC/R0 \#PF access handler. */
2086 STAMPROFILE StatMonitorPfRZ;
2087 /** Profiling the RC/R0 access we've handled (except REP STOSD). */
2088 STAMPROFILE StatMonitorPfRZHandled;
2089 /** Times we've failed interpreting the instruction. */
2090 STAMCOUNTER StatMonitorPfRZEmulateInstr;
2091 /** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
2092 STAMPROFILE StatMonitorPfRZFlushPage;
2093 /** Times we've detected a page table reinit. */
2094 STAMCOUNTER StatMonitorPfRZFlushReinit;
2095 /** Counting flushes for pages that are modified too often. */
2096 STAMCOUNTER StatMonitorPfRZFlushModOverflow;
2097 /** Times we've detected fork(). */
2098 STAMCOUNTER StatMonitorPfRZFork;
2099 /** Times we've failed interpreting a patch code instruction. */
2100 STAMCOUNTER StatMonitorPfRZIntrFailPatch1;
2101 /** Times we've failed interpreting a patch code instruction during flushing. */
2102 STAMCOUNTER StatMonitorPfRZIntrFailPatch2;
2103 /** The number of times we've seen rep prefixes we can't handle. */
2104 STAMCOUNTER StatMonitorPfRZRepPrefix;
2105 /** Profiling the REP STOSD cases we've handled. */
2106 STAMPROFILE StatMonitorPfRZRepStosd;
2107
2108 /** Profiling the R0/RC regular access handler. */
2109 STAMPROFILE StatMonitorRZ;
2110 /** Profiling the pgmPoolFlushPage calls made from the regular access handler in R0/RC. */
2111 STAMPROFILE StatMonitorRZFlushPage;
2112 /** Per access size counts indexed by size minus 1, last for larger. */
2113 STAMCOUNTER aStatMonitorRZSizes[16+3];
2114 /** Missaligned access counts indexed by offset - 1. */
2115 STAMCOUNTER aStatMonitorRZMisaligned[7];
2116
2117 /** Nr of handled PT faults. */
2118 STAMCOUNTER StatMonitorRZFaultPT;
2119 /** Nr of handled PD faults. */
2120 STAMCOUNTER StatMonitorRZFaultPD;
2121 /** Nr of handled PDPT faults. */
2122 STAMCOUNTER StatMonitorRZFaultPDPT;
2123 /** Nr of handled PML4 faults. */
2124 STAMCOUNTER StatMonitorRZFaultPML4;
2125
2126 /** Profiling the R3 access handler. */
2127 STAMPROFILE StatMonitorR3;
2128 /** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
2129 STAMPROFILE StatMonitorR3FlushPage;
2130 /** Per access size counts indexed by size minus 1, last for larger. */
2131 STAMCOUNTER aStatMonitorR3Sizes[16+3];
2132 /** Missaligned access counts indexed by offset - 1. */
2133 STAMCOUNTER aStatMonitorR3Misaligned[7];
2134 /** Nr of handled PT faults. */
2135 STAMCOUNTER StatMonitorR3FaultPT;
2136 /** Nr of handled PD faults. */
2137 STAMCOUNTER StatMonitorR3FaultPD;
2138 /** Nr of handled PDPT faults. */
2139 STAMCOUNTER StatMonitorR3FaultPDPT;
2140 /** Nr of handled PML4 faults. */
2141 STAMCOUNTER StatMonitorR3FaultPML4;
2142
2143 /** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
2144 STAMCOUNTER StatResetDirtyPages;
2145 /** Times we've called pgmPoolAddDirtyPage. */
2146 STAMCOUNTER StatDirtyPage;
2147 /** Times we've had to flush duplicates for dirty page management. */
2148 STAMCOUNTER StatDirtyPageDupFlush;
2149 /** Times we've had to flush because of overflow. */
2150 STAMCOUNTER StatDirtyPageOverFlowFlush;
2151
2152 /** The high water mark for cModifiedPages. */
2153 uint16_t cModifiedPagesHigh;
2154 uint16_t Alignment2[3]; /**< Align the next member on a 64-bit boundary. */
2155
2156 /** The number of cache hits. */
2157 STAMCOUNTER StatCacheHits;
2158 /** The number of cache misses. */
2159 STAMCOUNTER StatCacheMisses;
2160 /** The number of times we've got a conflict of 'kind' in the cache. */
2161 STAMCOUNTER StatCacheKindMismatches;
2162 /** Number of times we've been out of pages. */
2163 STAMCOUNTER StatCacheFreeUpOne;
2164 /** The number of cacheable allocations. */
2165 STAMCOUNTER StatCacheCacheable;
2166 /** The number of uncacheable allocations. */
2167 STAMCOUNTER StatCacheUncacheable;
2168#else
2169 uint32_t Alignment3; /**< Align the next member on a 64-bit boundary. */
2170#endif
2171 /** Profiling PGMR0PoolGrow(). */
2172 STAMPROFILE StatGrow;
2173 /** The AVL tree for looking up a page by its HC physical address. */
2174 AVLOHCPHYSTREE HCPhysTree;
2175 uint32_t Alignment4; /**< Align the next member on a 64-bit boundary. */
2176 /** Array of pages. (cMaxPages in length)
2177 * The Id is the index into thist array.
2178 */
2179 PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
2180} PGMPOOL, *PPGMPOOL, **PPPGMPOOL;
2181AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
2182AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
2183AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
2184#ifdef VBOX_WITH_STATISTICS
2185AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
2186#endif
2187AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
2188
2189
2190/** @def PGMPOOL_PAGE_2_PTR
2191 * Maps a pool page pool into the current context.
2192 *
2193 * @returns VBox status code.
2194 * @param a_pVM Pointer to the VM.
2195 * @param a_pPage The pool page.
2196 *
2197 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2198 * small page window employeed by that function. Be careful.
2199 * @remark There is no need to assert on the result.
2200 */
2201#if defined(VBOX_STRICT) || 1 /* temporarily going strict here */
2202# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage) pgmPoolMapPageStrict(a_pPage, __FUNCTION__)
2203DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE a_pPage, const char *pszCaller)
2204{
2205 RT_NOREF(pszCaller);
2206 AssertPtr(a_pPage);
2207 AssertMsg(RT_VALID_PTR(a_pPage->CTX_SUFF(pvPage)),
2208 ("enmKind=%d idx=%#x HCPhys=%RHp GCPhys=%RGp pvPageR3=%p pvPageR0=%p caller=%s\n",
2209 a_pPage->enmKind, a_pPage->idx, a_pPage->Core.Key, a_pPage->GCPhys, a_pPage->pvPageR3, a_pPage->pvPageR0, pszCaller));
2210 return a_pPage->CTX_SUFF(pvPage);
2211}
2212#else
2213# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage) ((a_pPage)->CTX_SUFF(pvPage))
2214#endif
2215
2216
2217/** @def PGMPOOL_PAGE_2_PTR_V2
2218 * Maps a pool page pool into the current context, taking both VM and VMCPU.
2219 *
2220 * @returns VBox status code.
2221 * @param a_pVM Pointer to the VM.
2222 * @param a_pVCpu The current CPU.
2223 * @param a_pPage The pool page.
2224 *
2225 * @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2226 * small page window employeed by that function. Be careful.
2227 * @remark There is no need to assert on the result.
2228 */
2229#define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage) PGMPOOL_PAGE_2_PTR((a_pVM), (a_pPage))
2230
2231
2232/** @name Per guest page tracking data.
2233 * This is currently as a 16-bit word in the PGMPAGE structure, the idea though
2234 * is to use more bits for it and split it up later on. But for now we'll play
2235 * safe and change as little as possible.
2236 *
2237 * The 16-bit word has two parts:
2238 *
2239 * The first 14-bit forms the @a idx field. It is either the index of a page in
2240 * the shadow page pool, or and index into the extent list.
2241 *
2242 * The 2 topmost bits makes up the @a cRefs field, which counts the number of
2243 * shadow page pool references to the page. If cRefs equals
2244 * PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
2245 * (misnomer) table and not the shadow page pool.
2246 *
2247 * See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
2248 * the 16-bit word.
2249 *
2250 * @{ */
2251/** The shift count for getting to the cRefs part. */
2252#define PGMPOOL_TD_CREFS_SHIFT 14
2253/** The mask applied after shifting the tracking data down by
2254 * PGMPOOL_TD_CREFS_SHIFT. */
2255#define PGMPOOL_TD_CREFS_MASK 0x3
2256/** The cRefs value used to indicate that the idx is the head of a
2257 * physical cross reference list. */
2258#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
2259/** The shift used to get idx. */
2260#define PGMPOOL_TD_IDX_SHIFT 0
2261/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
2262#define PGMPOOL_TD_IDX_MASK 0x3fff
2263/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
2264 * simply too many mappings of this page. */
2265#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
2266
2267/** @def PGMPOOL_TD_MAKE
2268 * Makes a 16-bit tracking data word.
2269 *
2270 * @returns tracking data.
2271 * @param cRefs The @a cRefs field. Must be within bounds!
2272 * @param idx The @a idx field. Must also be within bounds! */
2273#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) | (idx) )
2274
2275/** @def PGMPOOL_TD_GET_CREFS
2276 * Get the @a cRefs field from a tracking data word.
2277 *
2278 * @returns The @a cRefs field
2279 * @param u16 The tracking data word.
2280 * @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a
2281 * non-zero @a u16. */
2282#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
2283
2284/** @def PGMPOOL_TD_GET_IDX
2285 * Get the @a idx field from a tracking data word.
2286 *
2287 * @returns The @a idx field
2288 * @param u16 The tracking data word. */
2289#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
2290/** @} */
2291
2292
2293
2294/** @name A20 gate macros
2295 * @{ */
2296#define PGM_WITH_A20
2297#ifdef PGM_WITH_A20
2298# define PGM_A20_IS_ENABLED(a_pVCpu) ((a_pVCpu)->pgm.s.fA20Enabled)
2299# define PGM_A20_APPLY(a_pVCpu, a_GCPhys) ((a_GCPhys) & (a_pVCpu)->pgm.s.GCPhysA20Mask)
2300# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar) \
2301 do { a_GCPhysVar &= (a_pVCpu)->pgm.s.GCPhysA20Mask; } while (0)
2302# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys) Assert(PGM_A20_APPLY(pVCpu, a_GCPhys) == (a_GCPhys))
2303#else
2304# define PGM_A20_IS_ENABLED(a_pVCpu) (true)
2305# define PGM_A20_APPLY(a_pVCpu, a_GCPhys) (a_GCPhys)
2306# define PGM_A20_APPLY_TO_VAR(a_pVCpu, a_GCPhysVar) do { } while (0)
2307# define PGM_A20_ASSERT_MASKED(pVCpu, a_GCPhys) do { } while (0)
2308#endif
2309/** @} */
2310
2311
2312/**
2313 * Roots and anchors for trees and list employing self relative offsets as
2314 * pointers.
2315 *
2316 * When using self-relative offsets instead of pointers, the offsets needs to be
2317 * the same in all offsets. Thus the roots and anchors needs to live on the
2318 * hyper heap just like the nodes.
2319 */
2320typedef struct PGMTREES
2321{
2322 /** List of physical access handler types (offset pointers) of type
2323 * PGMPHYSHANDLERTYPEINT. This is needed for relocations. */
2324 RTLISTOFF32ANCHOR HeadPhysHandlerTypes;
2325 /** Physical access handlers (AVL range+offsetptr tree). */
2326 AVLROGCPHYSTREE PhysHandlers;
2327} PGMTREES;
2328/** Pointer to PGM trees. */
2329typedef PGMTREES *PPGMTREES;
2330
2331
2332/**
2333 * Guest page table walk for the AMD64 mode.
2334 */
2335typedef struct PGMPTWALKGSTAMD64
2336{
2337 PX86PML4 pPml4;
2338 PX86PML4E pPml4e;
2339 X86PML4E Pml4e;
2340
2341 PX86PDPT pPdpt;
2342 PX86PDPE pPdpe;
2343 X86PDPE Pdpe;
2344
2345 PX86PDPAE pPd;
2346 PX86PDEPAE pPde;
2347 X86PDEPAE Pde;
2348
2349 PX86PTPAE pPt;
2350 PX86PTEPAE pPte;
2351 X86PTEPAE Pte;
2352} PGMPTWALKGSTAMD64;
2353/** Pointer to a AMD64 guest page table walk. */
2354typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64;
2355/** Pointer to a const AMD64 guest page table walk. */
2356typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64;
2357
2358/**
2359 * Guest page table walk for the EPT mode.
2360 */
2361typedef struct PGMPTWALKGSTEPT
2362{
2363 PEPTPML4 pPml4;
2364 PEPTPML4E pPml4e;
2365 EPTPML4E Pml4e;
2366
2367 PEPTPDPT pPdpt;
2368 PEPTPDPTE pPdpte;
2369 EPTPDPTE Pdpte;
2370
2371 PEPTPD pPd;
2372 PEPTPDE pPde;
2373 EPTPDE Pde;
2374
2375 PEPTPT pPt;
2376 PEPTPTE pPte;
2377 EPTPTE Pte;
2378} PGMPTWALKGSTEPT;
2379/** Pointer to an EPT guest page table walk. */
2380typedef PGMPTWALKGSTEPT *PPGMPTWALKGSTEPT;
2381/** Pointer to a const EPT guest page table walk. */
2382typedef PGMPTWALKGSTEPT const *PCPGMPTWALKGSTEPT;
2383
2384/**
2385 * Guest page table walk for the PAE mode.
2386 */
2387typedef struct PGMPTWALKGSTPAE
2388{
2389 PX86PDPT pPdpt;
2390 PX86PDPE pPdpe;
2391 X86PDPE Pdpe;
2392
2393 PX86PDPAE pPd;
2394 PX86PDEPAE pPde;
2395 X86PDEPAE Pde;
2396
2397 PX86PTPAE pPt;
2398 PX86PTEPAE pPte;
2399 X86PTEPAE Pte;
2400} PGMPTWALKGSTPAE;
2401/** Pointer to a PAE guest page table walk. */
2402typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE;
2403/** Pointer to a const AMD64 guest page table walk. */
2404typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE;
2405
2406/**
2407 * Guest page table walk for the 32-bit mode.
2408 */
2409typedef struct PGMPTWALKGST32BIT
2410{
2411 PX86PD pPd;
2412 PX86PDE pPde;
2413 X86PDE Pde;
2414
2415 PX86PT pPt;
2416 PX86PTE pPte;
2417 X86PTE Pte;
2418} PGMPTWALKGST32BIT;
2419/** Pointer to a 32-bit guest page table walk. */
2420typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT;
2421/** Pointer to a const 32-bit guest page table walk. */
2422typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT;
2423
2424/**
2425 * Which part of PGMPTWALKGST that is valid.
2426 */
2427typedef enum PGMPTWALKGSTTYPE
2428{
2429 /** Customary invalid 0 value. */
2430 PGMPTWALKGSTTYPE_INVALID = 0,
2431 /** PGMPTWALKGST::u.Amd64 is valid. */
2432 PGMPTWALKGSTTYPE_AMD64,
2433 /** PGMPTWALKGST::u.Pae is valid. */
2434 PGMPTWALKGSTTYPE_PAE,
2435 /** PGMPTWALKGST::u.Legacy is valid. */
2436 PGMPTWALKGSTTYPE_32BIT,
2437 /** PGMPTWALKGST::u.Ept is valid. */
2438 PGMPTWALKGSTTYPE_EPT,
2439 /** Customary 32-bit type hack. */
2440 PGMPTWALKGSTTYPE_32BIT_HACK = 0x7fff0000
2441} PGMPTWALKGSTTYPE;
2442
2443/**
2444 * Combined guest page table walk result.
2445 */
2446typedef struct PGMPTWALKGST
2447{
2448 union
2449 {
2450 /** The page walker for AMD64. */
2451 PGMPTWALKGSTAMD64 Amd64;
2452 /** The page walker for PAE (32-bit). */
2453 PGMPTWALKGSTPAE Pae;
2454 /** The page walker for 32-bit paging (called legacy due to C naming
2455 * convension). */
2456 PGMPTWALKGST32BIT Legacy;
2457 /** The page walker for EPT (SLAT). */
2458 PGMPTWALKGSTEPT Ept;
2459 } u;
2460 /** Indicates which part of the union is valid. */
2461 PGMPTWALKGSTTYPE enmType;
2462} PGMPTWALKGST;
2463/** Pointer to a combined guest page table walk result. */
2464typedef PGMPTWALKGST *PPGMPTWALKGST;
2465/** Pointer to a read-only combined guest page table walk result. */
2466typedef PGMPTWALKGST const *PCPGMPTWALKGST;
2467
2468
2469/** @name Paging mode macros
2470 * @{
2471 */
2472#ifdef IN_RING3
2473# define PGM_CTX(a,b) a##R3##b
2474# define PGM_CTX_STR(a,b) a "R3" b
2475# define PGM_CTX_DECL(type) DECLCALLBACK(type)
2476#elif defined(IN_RING0)
2477# define PGM_CTX(a,b) a##R0##b
2478# define PGM_CTX_STR(a,b) a "R0" b
2479# define PGM_CTX_DECL(type) VMMDECL(type)
2480#else
2481# error "Not IN_RING3 or IN_RING0!"
2482#endif
2483
2484#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
2485#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
2486#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
2487#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
2488#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
2489#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
2490#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
2491#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
2492#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
2493#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
2494#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
2495#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
2496#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
2497#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
2498#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
2499#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
2500
2501#define PGM_GST_SLAT_NAME_EPT(name) PGM_CTX(pgm,GstSlatEpt##name)
2502#define PGM_GST_SLAT_NAME_RC_EPT_STR(name) "pgmRCGstSlatEpt" #name
2503#define PGM_GST_SLAT_NAME_R0_EPT_STR(name) "pgmR0GstSlatEpt" #name
2504#define PGM_GST_SLAT_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_SLAT_NAME(name)
2505
2506#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
2507#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
2508#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
2509#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
2510#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
2511#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
2512#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
2513#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
2514#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
2515#define PGM_SHW_NAME_NESTED_32BIT(name) PGM_CTX(pgm,ShwNested32Bit##name)
2516#define PGM_SHW_NAME_RC_NESTED_32BIT_STR(name) "pgmRCShwNested32Bit" #name
2517#define PGM_SHW_NAME_R0_NESTED_32BIT_STR(name) "pgmR0ShwNested32Bit" #name
2518#define PGM_SHW_NAME_NESTED_PAE(name) PGM_CTX(pgm,ShwNestedPAE##name)
2519#define PGM_SHW_NAME_RC_NESTED_PAE_STR(name) "pgmRCShwNestedPAE" #name
2520#define PGM_SHW_NAME_R0_NESTED_PAE_STR(name) "pgmR0ShwNestedPAE" #name
2521#define PGM_SHW_NAME_NESTED_AMD64(name) PGM_CTX(pgm,ShwNestedAMD64##name)
2522#define PGM_SHW_NAME_RC_NESTED_AMD64_STR(name) "pgmRCShwNestedAMD64" #name
2523#define PGM_SHW_NAME_R0_NESTED_AMD64_STR(name) "pgmR0ShwNestedAMD64" #name
2524#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2525#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2526#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2527#define PGM_SHW_NAME_NONE(name) PGM_CTX(pgm,ShwNone##name)
2528#define PGM_SHW_NAME_RC_NONE_STR(name) "pgmRCShwNone" #name
2529#define PGM_SHW_NAME_R0_NONE_STR(name) "pgmR0ShwNone" #name
2530#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2531
2532/* Shw_Gst */
2533#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2534#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2535#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2536#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2537#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2538#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2539#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2540#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2541#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2542#define PGM_BTH_NAME_NESTED_32BIT_REAL(name) PGM_CTX(pgm,BthNested32BitReal##name)
2543#define PGM_BTH_NAME_NESTED_32BIT_PROT(name) PGM_CTX(pgm,BthNested32BitProt##name)
2544#define PGM_BTH_NAME_NESTED_32BIT_32BIT(name) PGM_CTX(pgm,BthNested32Bit32Bit##name)
2545#define PGM_BTH_NAME_NESTED_32BIT_PAE(name) PGM_CTX(pgm,BthNested32BitPAE##name)
2546#define PGM_BTH_NAME_NESTED_32BIT_AMD64(name) PGM_CTX(pgm,BthNested32BitAMD64##name)
2547#define PGM_BTH_NAME_NESTED_PAE_REAL(name) PGM_CTX(pgm,BthNestedPAEReal##name)
2548#define PGM_BTH_NAME_NESTED_PAE_PROT(name) PGM_CTX(pgm,BthNestedPAEProt##name)
2549#define PGM_BTH_NAME_NESTED_PAE_32BIT(name) PGM_CTX(pgm,BthNestedPAE32Bit##name)
2550#define PGM_BTH_NAME_NESTED_PAE_PAE(name) PGM_CTX(pgm,BthNestedPAEPAE##name)
2551#define PGM_BTH_NAME_NESTED_PAE_AMD64(name) PGM_CTX(pgm,BthNestedPAEAMD64##name)
2552#define PGM_BTH_NAME_NESTED_AMD64_REAL(name) PGM_CTX(pgm,BthNestedAMD64Real##name)
2553#define PGM_BTH_NAME_NESTED_AMD64_PROT(name) PGM_CTX(pgm,BthNestedAMD64Prot##name)
2554#define PGM_BTH_NAME_NESTED_AMD64_32BIT(name) PGM_CTX(pgm,BthNestedAMD6432Bit##name)
2555#define PGM_BTH_NAME_NESTED_AMD64_PAE(name) PGM_CTX(pgm,BthNestedAMD64PAE##name)
2556#define PGM_BTH_NAME_NESTED_AMD64_AMD64(name) PGM_CTX(pgm,BthNestedAMD64AMD64##name)
2557#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2558#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2559#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2560#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2561#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2562#define PGM_BTH_NAME_NONE_REAL(name) PGM_CTX(pgm,BthNoneReal##name)
2563#define PGM_BTH_NAME_NONE_PROT(name) PGM_CTX(pgm,BthNoneProt##name)
2564#define PGM_BTH_NAME_NONE_32BIT(name) PGM_CTX(pgm,BthNone32Bit##name)
2565#define PGM_BTH_NAME_NONE_PAE(name) PGM_CTX(pgm,BthNonePAE##name)
2566#define PGM_BTH_NAME_NONE_AMD64(name) PGM_CTX(pgm,BthNoneAMD64##name)
2567
2568#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2569#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2570#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2571#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2572#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2573#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2574#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2575#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2576#define PGM_BTH_NAME_RC_NESTED_32BIT_REAL_STR(name) "pgmRCBthNested32BitReal" #name
2577#define PGM_BTH_NAME_RC_NESTED_32BIT_PROT_STR(name) "pgmRCBthNested32BitProt" #name
2578#define PGM_BTH_NAME_RC_NESTED_32BIT_32BIT_STR(name) "pgmRCBthNested32Bit32Bit" #name
2579#define PGM_BTH_NAME_RC_NESTED_32BIT_PAE_STR(name) "pgmRCBthNested32BitPAE" #name
2580#define PGM_BTH_NAME_RC_NESTED_32BIT_AMD64_STR(name) "pgmRCBthNested32BitAMD64" #name
2581#define PGM_BTH_NAME_RC_NESTED_PAE_REAL_STR(name) "pgmRCBthNestedPAEReal" #name
2582#define PGM_BTH_NAME_RC_NESTED_PAE_PROT_STR(name) "pgmRCBthNestedPAEProt" #name
2583#define PGM_BTH_NAME_RC_NESTED_PAE_32BIT_STR(name) "pgmRCBthNestedPAE32Bit" #name
2584#define PGM_BTH_NAME_RC_NESTED_PAE_PAE_STR(name) "pgmRCBthNestedPAEPAE" #name
2585#define PGM_BTH_NAME_RC_NESTED_PAE_AMD64_STR(name) "pgmRCBthNestedPAEAMD64" #name
2586#define PGM_BTH_NAME_RC_NESTED_AMD64_REAL_STR(name) "pgmRCBthNestedAMD64Real" #name
2587#define PGM_BTH_NAME_RC_NESTED_AMD64_PROT_STR(name) "pgmRCBthNestedAMD64Prot" #name
2588#define PGM_BTH_NAME_RC_NESTED_AMD64_32BIT_STR(name) "pgmRCBthNestedAMD6432Bit" #name
2589#define PGM_BTH_NAME_RC_NESTED_AMD64_PAE_STR(name) "pgmRCBthNestedAMD64PAE" #name
2590#define PGM_BTH_NAME_RC_NESTED_AMD64_AMD64_STR(name) "pgmRCBthNestedAMD64AMD64" #name
2591#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2592#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2593#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2594#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2595#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2596
2597#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2598#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2599#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2600#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2601#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2602#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2603#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2604#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2605#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2606#define PGM_BTH_NAME_R0_NESTED_32BIT_REAL_STR(name) "pgmR0BthNested32BitReal" #name
2607#define PGM_BTH_NAME_R0_NESTED_32BIT_PROT_STR(name) "pgmR0BthNested32BitProt" #name
2608#define PGM_BTH_NAME_R0_NESTED_32BIT_32BIT_STR(name) "pgmR0BthNested32Bit32Bit" #name
2609#define PGM_BTH_NAME_R0_NESTED_32BIT_PAE_STR(name) "pgmR0BthNested32BitPAE" #name
2610#define PGM_BTH_NAME_R0_NESTED_32BIT_AMD64_STR(name) "pgmR0BthNested32BitAMD64" #name
2611#define PGM_BTH_NAME_R0_NESTED_PAE_REAL_STR(name) "pgmR0BthNestedPAEReal" #name
2612#define PGM_BTH_NAME_R0_NESTED_PAE_PROT_STR(name) "pgmR0BthNestedPAEProt" #name
2613#define PGM_BTH_NAME_R0_NESTED_PAE_32BIT_STR(name) "pgmR0BthNestedPAE32Bit" #name
2614#define PGM_BTH_NAME_R0_NESTED_PAE_PAE_STR(name) "pgmR0BthNestedPAEPAE" #name
2615#define PGM_BTH_NAME_R0_NESTED_PAE_AMD64_STR(name) "pgmR0BthNestedPAEAMD64" #name
2616#define PGM_BTH_NAME_R0_NESTED_AMD64_REAL_STR(name) "pgmR0BthNestedAMD64Real" #name
2617#define PGM_BTH_NAME_R0_NESTED_AMD64_PROT_STR(name) "pgmR0BthNestedAMD64Prot" #name
2618#define PGM_BTH_NAME_R0_NESTED_AMD64_32BIT_STR(name) "pgmR0BthNestedAMD6432Bit" #name
2619#define PGM_BTH_NAME_R0_NESTED_AMD64_PAE_STR(name) "pgmR0BthNestedAMD64PAE" #name
2620#define PGM_BTH_NAME_R0_NESTED_AMD64_AMD64_STR(name) "pgmR0BthNestedAMD64AMD64" #name
2621#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2622#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2623#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2624#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2625#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2626
2627#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2628/** @} */
2629
2630
2631/**
2632 * Function pointers for guest paging.
2633 */
2634typedef struct PGMMODEDATAGST
2635{
2636 /** The guest mode type. */
2637 uint32_t uType;
2638 DECLCALLBACKMEMBER(int, pfnGetPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk));
2639 DECLCALLBACKMEMBER(int, pfnModifyPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2640 DECLCALLBACKMEMBER(int, pfnEnter,(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3));
2641 DECLCALLBACKMEMBER(int, pfnExit,(PVMCPUCC pVCpu));
2642#ifdef IN_RING3
2643 DECLCALLBACKMEMBER(int, pfnRelocate,(PVMCPUCC pVCpu, RTGCPTR offDelta)); /**< Only in ring-3. */
2644#endif
2645} PGMMODEDATAGST;
2646
2647/** The length of g_aPgmGuestModeData. */
2648#if VBOX_WITH_64_BITS_GUESTS
2649# define PGM_GUEST_MODE_DATA_ARRAY_SIZE (PGM_TYPE_AMD64 + 1)
2650#else
2651# define PGM_GUEST_MODE_DATA_ARRAY_SIZE (PGM_TYPE_PAE + 1)
2652#endif
2653/** The guest mode data array. */
2654extern PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE];
2655
2656
2657/**
2658 * Function pointers for shadow paging.
2659 */
2660typedef struct PGMMODEDATASHW
2661{
2662 /** The shadow mode type. */
2663 uint32_t uType;
2664 DECLCALLBACKMEMBER(int, pfnGetPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2665 DECLCALLBACKMEMBER(int, pfnModifyPage,(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags,
2666 uint64_t fMask, uint32_t fOpFlags));
2667 DECLCALLBACKMEMBER(int, pfnEnter,(PVMCPUCC pVCpu, bool fIs64BitsPagingMode));
2668 DECLCALLBACKMEMBER(int, pfnExit,(PVMCPUCC pVCpu));
2669#ifdef IN_RING3
2670 DECLCALLBACKMEMBER(int, pfnRelocate,(PVMCPUCC pVCpu, RTGCPTR offDelta)); /**< Only in ring-3. */
2671#endif
2672} PGMMODEDATASHW;
2673
2674/** The length of g_aPgmShadowModeData. */
2675#define PGM_SHADOW_MODE_DATA_ARRAY_SIZE PGM_TYPE_END
2676/** The shadow mode data array. */
2677extern PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE];
2678
2679
2680/**
2681 * Function pointers for guest+shadow paging.
2682 */
2683typedef struct PGMMODEDATABTH
2684{
2685 /** The shadow mode type. */
2686 uint32_t uShwType;
2687 /** The guest mode type. */
2688 uint32_t uGstType;
2689
2690 DECLCALLBACKMEMBER(int, pfnInvalidatePage,(PVMCPUCC pVCpu, RTGCPTR GCPtrPage));
2691 DECLCALLBACKMEMBER(int, pfnSyncCR3,(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2692 DECLCALLBACKMEMBER(int, pfnPrefetchPage,(PVMCPUCC pVCpu, RTGCPTR GCPtrPage));
2693 DECLCALLBACKMEMBER(int, pfnVerifyAccessSyncPage,(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2694 DECLCALLBACKMEMBER(int, pfnMapCR3,(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3));
2695 DECLCALLBACKMEMBER(int, pfnUnmapCR3,(PVMCPUCC pVCpu));
2696 DECLCALLBACKMEMBER(int, pfnEnter,(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3));
2697#ifndef IN_RING3
2698 DECLCALLBACKMEMBER(int, pfnTrap0eHandler,(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
2699#endif
2700#ifdef VBOX_STRICT
2701 DECLCALLBACKMEMBER(unsigned, pfnAssertCR3,(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2702#endif
2703} PGMMODEDATABTH;
2704
2705/** The length of g_aPgmBothModeData. */
2706#define PGM_BOTH_MODE_DATA_ARRAY_SIZE ((PGM_TYPE_END - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END)
2707/** The guest+shadow mode data array. */
2708extern PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE];
2709
2710
2711#ifdef VBOX_WITH_STATISTICS
2712/**
2713 * PGM statistics.
2714 */
2715typedef struct PGMSTATS
2716{
2717 /* R3 only: */
2718 STAMCOUNTER StatR3DetectedConflicts; /**< R3: Number of times PGMR3MapHasConflicts() detected a conflict. */
2719 STAMPROFILE StatR3ResolveConflict; /**< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). */
2720
2721 /* R3+RZ */
2722 STAMCOUNTER StatRZChunkR3MapTlbHits; /**< RC/R0: Ring-3/0 chunk mapper TLB hits. */
2723 STAMCOUNTER StatRZChunkR3MapTlbMisses; /**< RC/R0: Ring-3/0 chunk mapper TLB misses. */
2724 STAMCOUNTER StatRZPageMapTlbHits; /**< RC/R0: Ring-3/0 page mapper TLB hits. */
2725 STAMCOUNTER StatRZPageMapTlbMisses; /**< RC/R0: Ring-3/0 page mapper TLB misses. */
2726 STAMCOUNTER StatPageMapTlbFlushes; /**< ALL: Ring-3/0 page mapper TLB flushes. */
2727 STAMCOUNTER StatPageMapTlbFlushEntry; /**< ALL: Ring-3/0 page mapper TLB flushes. */
2728 STAMCOUNTER StatR3ChunkR3MapTlbHits; /**< R3: Ring-3/0 chunk mapper TLB hits. */
2729 STAMCOUNTER StatR3ChunkR3MapTlbMisses; /**< R3: Ring-3/0 chunk mapper TLB misses. */
2730 STAMCOUNTER StatR3PageMapTlbHits; /**< R3: Ring-3/0 page mapper TLB hits. */
2731 STAMCOUNTER StatR3PageMapTlbMisses; /**< R3: Ring-3/0 page mapper TLB misses. */
2732 STAMCOUNTER StatRZRamRangeTlbHits; /**< RC/R0: RAM range TLB hits. */
2733 STAMCOUNTER StatRZRamRangeTlbMisses; /**< RC/R0: RAM range TLB misses. */
2734 STAMCOUNTER StatR3RamRangeTlbHits; /**< R3: RAM range TLB hits. */
2735 STAMCOUNTER StatR3RamRangeTlbMisses; /**< R3: RAM range TLB misses. */
2736 STAMCOUNTER StatR3PhysHandlerReset; /**< R3: The number of times PGMHandlerPhysicalReset is called. */
2737 STAMCOUNTER StatRZPhysHandlerReset; /**< RC/R0: The number of times PGMHandlerPhysicalReset is called. */
2738 STAMCOUNTER StatR3PhysHandlerLookupHits; /**< R3: Number of cache hits when looking up physical handlers. */
2739 STAMCOUNTER StatR3PhysHandlerLookupMisses; /**< R3: Number of cache misses when looking up physical handlers. */
2740 STAMCOUNTER StatRZPhysHandlerLookupHits; /**< RC/R0: Number of cache hits when lookup up physical handlers. */
2741 STAMCOUNTER StatRZPhysHandlerLookupMisses; /**< RC/R0: Number of cache misses when looking up physical handlers */
2742 STAMCOUNTER StatRZPageReplaceShared; /**< RC/R0: Times a shared page has been replaced by a private one. */
2743 STAMCOUNTER StatRZPageReplaceZero; /**< RC/R0: Times the zero page has been replaced by a private one. */
2744/// @todo STAMCOUNTER StatRZPageHandyAllocs; /**< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. */
2745 STAMCOUNTER StatR3PageReplaceShared; /**< R3: Times a shared page has been replaced by a private one. */
2746 STAMCOUNTER StatR3PageReplaceZero; /**< R3: Times the zero page has been replaced by a private one. */
2747/// @todo STAMCOUNTER StatR3PageHandyAllocs; /**< R3: The number of times we've executed GMMR3AllocateHandyPages. */
2748
2749 /* RC only: */
2750 STAMCOUNTER StatRCInvlPgConflict; /**< RC: Number of times PGMInvalidatePage() detected a mapping conflict. */
2751 STAMCOUNTER StatRCInvlPgSyncMonCR3; /**< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. */
2752
2753 STAMCOUNTER StatRZPhysRead;
2754 STAMCOUNTER StatRZPhysReadBytes;
2755 STAMCOUNTER StatRZPhysWrite;
2756 STAMCOUNTER StatRZPhysWriteBytes;
2757 STAMCOUNTER StatR3PhysRead;
2758 STAMCOUNTER StatR3PhysReadBytes;
2759 STAMCOUNTER StatR3PhysWrite;
2760 STAMCOUNTER StatR3PhysWriteBytes;
2761 STAMCOUNTER StatRCPhysRead;
2762 STAMCOUNTER StatRCPhysReadBytes;
2763 STAMCOUNTER StatRCPhysWrite;
2764 STAMCOUNTER StatRCPhysWriteBytes;
2765
2766 STAMCOUNTER StatRZPhysSimpleRead;
2767 STAMCOUNTER StatRZPhysSimpleReadBytes;
2768 STAMCOUNTER StatRZPhysSimpleWrite;
2769 STAMCOUNTER StatRZPhysSimpleWriteBytes;
2770 STAMCOUNTER StatR3PhysSimpleRead;
2771 STAMCOUNTER StatR3PhysSimpleReadBytes;
2772 STAMCOUNTER StatR3PhysSimpleWrite;
2773 STAMCOUNTER StatR3PhysSimpleWriteBytes;
2774 STAMCOUNTER StatRCPhysSimpleRead;
2775 STAMCOUNTER StatRCPhysSimpleReadBytes;
2776 STAMCOUNTER StatRCPhysSimpleWrite;
2777 STAMCOUNTER StatRCPhysSimpleWriteBytes;
2778
2779 STAMCOUNTER StatTrackVirgin; /**< The number of first time shadowings. */
2780 STAMCOUNTER StatTrackAliased; /**< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. */
2781 STAMCOUNTER StatTrackAliasedMany; /**< The number of times we're tracking using cRef2. */
2782 STAMCOUNTER StatTrackAliasedLots; /**< The number of times we're hitting pages which has overflowed cRef2. */
2783 STAMCOUNTER StatTrackNoExtentsLeft; /**< The number of times the extent list was exhausted. */
2784 STAMCOUNTER StatTrackOverflows; /**< The number of times the extent list grows to long. */
2785 STAMPROFILE StatTrackDeref; /**< Profiling of SyncPageWorkerTrackDeref (expensive). */
2786
2787 STAMPROFILE StatLargePageAlloc2; /**< Time spent setting up newly allocated large pages. */
2788 STAMPROFILE StatLargePageSetup; /**< Time spent setting up newly allocated large pages. */
2789 /** pgmPhysIsValidLargePage profiling - R3 */
2790 STAMPROFILE StatR3IsValidLargePage;
2791 /** pgmPhysIsValidLargePage profiling - RZ*/
2792 STAMPROFILE StatRZIsValidLargePage;
2793
2794 STAMPROFILE StatChunkAging;
2795 STAMPROFILE StatChunkFindCandidate;
2796 STAMPROFILE StatChunkUnmap;
2797 STAMPROFILE StatChunkMap;
2798} PGMSTATS;
2799#endif /* VBOX_WITH_STATISTICS */
2800
2801
2802/**
2803 * PGM Data (part of VM)
2804 */
2805typedef struct PGM
2806{
2807 /** @cfgm{/RamPreAlloc, boolean, false}
2808 * Indicates whether the base RAM should all be allocated before starting
2809 * the VM (default), or if it should be allocated when first written to.
2810 */
2811 bool fRamPreAlloc;
2812#ifdef VBOX_WITH_PGM_NEM_MODE
2813 /** Set if we're operating in NEM memory mode.
2814 *
2815 * NEM mode implies that memory is allocated in big chunks for each RAM range
2816 * rather than on demand page by page. Memory is also not locked and PGM has
2817 * therefore no physical addresses for them. Page sharing is out of the
2818 * question. Ballooning depends on the native execution engine, but probably
2819 * pointless as well. */
2820 bool fNemMode;
2821# define PGM_IS_IN_NEM_MODE(a_pVM) ((a_pVM)->pgm.s.fNemMode)
2822#else
2823# define PGM_IS_IN_NEM_MODE(a_pVM) (false)
2824#endif
2825 /** Indicates whether write monitoring is currently in use.
2826 * This is used to prevent conflicts between live saving and page sharing
2827 * detection. */
2828 bool fPhysWriteMonitoringEngaged;
2829 /** Set if the CPU has less than 52-bit physical address width.
2830 * This is used */
2831 bool fLessThan52PhysicalAddressBits;
2832 /** Set when nested paging is active.
2833 * This is meant to save calls to HMIsNestedPagingActive and let the
2834 * compilers optimize the code better. Whether we use nested paging or
2835 * not is something we find out during VMM initialization and we won't
2836 * change this later on. */
2837 bool fNestedPaging;
2838 /** We're not in a state which permits writes to guest memory.
2839 * (Only used in strict builds.) */
2840 bool fNoMorePhysWrites;
2841 /** @cfgm{/PageFusionAllowed, boolean, false}
2842 * Whether page fusion is allowed. */
2843 bool fPageFusionAllowed;
2844 /** @cfgm{/PGM/PciPassThrough, boolean, false}
2845 * Whether PCI passthrough is enabled. */
2846 bool fPciPassthrough;
2847 /** The number of MMIO2 regions (serves as the next MMIO2 ID). */
2848 uint8_t cMmio2Regions;
2849 /** Restore original ROM page content when resetting after loading state.
2850 * The flag is set by pgmR3LoadRomRanges and cleared at reset. This
2851 * enables the VM to start using an updated ROM without requiring powering
2852 * down the VM, just rebooting or resetting it. */
2853 bool fRestoreRomPagesOnReset;
2854 /** Whether to automatically clear all RAM pages on reset. */
2855 bool fZeroRamPagesOnReset;
2856 /** Large page enabled flag. */
2857 bool fUseLargePages;
2858 /** Alignment padding. */
2859#ifndef VBOX_WITH_PGM_NEM_MODE
2860 bool afAlignment3[1];
2861#endif
2862 /** The host paging mode. (This is what SUPLib reports.) */
2863 SUPPAGINGMODE enmHostMode;
2864 bool afAlignment3b[2];
2865
2866 /** Generation ID for the RAM ranges. This member is incremented everytime
2867 * a RAM range is linked or unlinked. */
2868 uint32_t volatile idRamRangesGen;
2869
2870 /** Physical access handler type for ROM protection. */
2871 PGMPHYSHANDLERTYPE hRomPhysHandlerType;
2872 /** Physical access handler type for MMIO2 dirty page tracing. */
2873 PGMPHYSHANDLERTYPE hMmio2DirtyPhysHandlerType;
2874
2875 /** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2876 RTGCPHYS GCPhys4MBPSEMask;
2877 /** Mask containing the invalid bits of a guest physical address.
2878 * @remarks this does not stop at bit 52. */
2879 RTGCPHYS GCPhysInvAddrMask;
2880
2881
2882 /** RAM range TLB for R3. */
2883 R3PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR3[PGM_RAMRANGE_TLB_ENTRIES];
2884 /** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2885 * This is sorted by physical address and contains no overlapping ranges. */
2886 R3PTRTYPE(PPGMRAMRANGE) pRamRangesXR3;
2887 /** Root of the RAM range search tree for ring-3. */
2888 R3PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR3;
2889 /** PGM offset based trees - R3 Ptr. */
2890 R3PTRTYPE(PPGMTREES) pTreesR3;
2891 /** Caching the last physical handler we looked up in R3. */
2892 R3PTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerR3;
2893 /** Shadow Page Pool - R3 Ptr. */
2894 R3PTRTYPE(PPGMPOOL) pPoolR3;
2895 /** Pointer to the list of ROM ranges - for R3.
2896 * This is sorted by physical address and contains no overlapping ranges. */
2897 R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2898 /** Pointer to the list of MMIO2 ranges - for R3.
2899 * Registration order. */
2900 R3PTRTYPE(PPGMREGMMIO2RANGE) pRegMmioRangesR3;
2901 /** MMIO2 lookup array for ring-3. Indexed by idMmio2 minus 1. */
2902 R3PTRTYPE(PPGMREGMMIO2RANGE) apMmio2RangesR3[PGM_MMIO2_MAX_RANGES];
2903
2904 /** RAM range TLB for R0. */
2905 R0PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR0[PGM_RAMRANGE_TLB_ENTRIES];
2906 /** R0 pointer corresponding to PGM::pRamRangesXR3. */
2907 R0PTRTYPE(PPGMRAMRANGE) pRamRangesXR0;
2908 /** Root of the RAM range search tree for ring-0. */
2909 R0PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR0;
2910 /** PGM offset based trees - R0 Ptr. */
2911 R0PTRTYPE(PPGMTREES) pTreesR0;
2912 /** Caching the last physical handler we looked up in R0. */
2913 R0PTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerR0;
2914 /** Shadow Page Pool - R0 Ptr. */
2915 R0PTRTYPE(PPGMPOOL) pPoolR0;
2916 /** R0 pointer corresponding to PGM::pRomRangesR3. */
2917 R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2918 /** MMIO2 lookup array for ring-0. Indexed by idMmio2 minus 1. */
2919 R0PTRTYPE(PPGMREGMMIO2RANGE) apMmio2RangesR0[PGM_MMIO2_MAX_RANGES];
2920
2921 /** Hack: Number of deprecated page mapping locks taken by the current lock
2922 * owner via pgmPhysGCPhys2CCPtrInternalDepr. */
2923 uint32_t cDeprecatedPageLocks;
2924 /** Alignment padding. */
2925 uint32_t au32Alignment2[1];
2926
2927 /** PGM critical section.
2928 * This protects the physical, ram ranges, and the page flag updating (some of
2929 * it anyway).
2930 */
2931 PDMCRITSECT CritSectX;
2932
2933 /**
2934 * Data associated with managing the ring-3 mappings of the allocation chunks.
2935 */
2936 struct
2937 {
2938 /** The chunk mapping TLB. */
2939 PGMCHUNKR3MAPTLB Tlb;
2940 /** The chunk tree, ordered by chunk id. */
2941 R3PTRTYPE(PAVLU32NODECORE) pTree;
2942#if HC_ARCH_BITS == 32
2943 uint32_t u32Alignment0;
2944#endif
2945 /** The number of mapped chunks. */
2946 uint32_t c;
2947 /** @cfgm{/PGM/MaxRing3Chunks, uint32_t, host dependent}
2948 * The maximum number of mapped chunks. On 64-bit this is unlimited by default,
2949 * on 32-bit it defaults to 1 or 3 GB depending on the host. */
2950 uint32_t cMax;
2951 /** The current time. This is incremented whenever a chunk is inserted. */
2952 uint32_t iNow;
2953 /** Alignment padding. */
2954 uint32_t au32Alignment1[3];
2955 } ChunkR3Map;
2956
2957 /** The page mapping TLB for ring-3. */
2958 PGMPAGER3MAPTLB PhysTlbR3;
2959 /** The page mapping TLB for ring-0. */
2960 PGMPAGER0MAPTLB PhysTlbR0;
2961
2962 /** @name The zero page.
2963 * @{ */
2964 /** The host physical address of the zero page. */
2965 RTHCPHYS HCPhysZeroPg;
2966 /** The ring-3 mapping of the zero page. */
2967 RTR3PTR pvZeroPgR3;
2968 /** The ring-0 mapping of the zero page. */
2969 RTR0PTR pvZeroPgR0;
2970 /** @}*/
2971
2972 /** @name The Invalid MMIO page.
2973 * This page is filled with 0xfeedface.
2974 * @{ */
2975 /** The host physical address of the invalid MMIO page. */
2976 RTHCPHYS HCPhysMmioPg;
2977 /** The host pysical address of the invalid MMIO page plus all invalid
2978 * physical address bits set. This is used to trigger X86_TRAP_PF_RSVD.
2979 * @remarks Check fLessThan52PhysicalAddressBits before use. */
2980 RTHCPHYS HCPhysInvMmioPg;
2981 /** The ring-3 mapping of the invalid MMIO page. */
2982 RTR3PTR pvMmioPgR3;
2983#if HC_ARCH_BITS == 32
2984 RTR3PTR R3PtrAlignment4;
2985#endif
2986 /** @} */
2987
2988
2989 /** The number of handy pages. */
2990 uint32_t cHandyPages;
2991
2992 /** The number of large handy pages. */
2993 uint32_t cLargeHandyPages;
2994
2995 /**
2996 * Array of handy pages.
2997 *
2998 * This array is used in a two way communication between pgmPhysAllocPage
2999 * and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
3000 * an intermediary.
3001 *
3002 * The size of this array is important, see pgmPhysEnsureHandyPage for details.
3003 * (The current size of 32 pages, means 128 KB of handy memory.)
3004 */
3005 GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
3006
3007 /**
3008 * Array of large handy pages. (currently size 1)
3009 *
3010 * This array is used in a two way communication between pgmPhysAllocLargePage
3011 * and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as
3012 * an intermediary.
3013 */
3014 GMMPAGEDESC aLargeHandyPage[1];
3015 /** When to try allocate large pages again after a failure. */
3016 uint64_t nsLargePageRetry;
3017 /** Number of repeated long allocation times. */
3018 uint32_t cLargePageLongAllocRepeats;
3019 uint32_t uPadding5;
3020
3021 /**
3022 * Live save data.
3023 */
3024 struct
3025 {
3026 /** Per type statistics. */
3027 struct
3028 {
3029 /** The number of ready pages. */
3030 uint32_t cReadyPages;
3031 /** The number of dirty pages. */
3032 uint32_t cDirtyPages;
3033 /** The number of ready zero pages. */
3034 uint32_t cZeroPages;
3035 /** The number of write monitored pages. */
3036 uint32_t cMonitoredPages;
3037 } Rom,
3038 Mmio2,
3039 Ram;
3040 /** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
3041 uint32_t cIgnoredPages;
3042 /** Indicates that a live save operation is active. */
3043 bool fActive;
3044 /** Padding. */
3045 bool afReserved[2];
3046 /** The next history index. */
3047 uint8_t iDirtyPagesHistory;
3048 /** History of the total amount of dirty pages. */
3049 uint32_t acDirtyPagesHistory[64];
3050 /** Short term dirty page average. */
3051 uint32_t cDirtyPagesShort;
3052 /** Long term dirty page average. */
3053 uint32_t cDirtyPagesLong;
3054 /** The number of saved pages. This is used to get some kind of estimate of the
3055 * link speed so we can decide when we're done. It is reset after the first
3056 * 7 passes so the speed estimate doesn't get inflated by the initial set of
3057 * zero pages. */
3058 uint64_t cSavedPages;
3059 /** The nanosecond timestamp when cSavedPages was 0. */
3060 uint64_t uSaveStartNS;
3061 /** Pages per second (for statistics). */
3062 uint32_t cPagesPerSecond;
3063 uint32_t cAlignment;
3064 } LiveSave;
3065
3066 /** @name Error injection.
3067 * @{ */
3068 /** Inject handy page allocation errors pretending we're completely out of
3069 * memory. */
3070 bool volatile fErrInjHandyPages;
3071 /** Padding. */
3072 bool afReserved[3];
3073 /** @} */
3074
3075 /** @name Release Statistics
3076 * @{ */
3077 uint32_t cAllPages; /**< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) */
3078 uint32_t cPrivatePages; /**< The number of private pages. */
3079 uint32_t cSharedPages; /**< The number of shared pages. */
3080 uint32_t cReusedSharedPages; /**< The number of reused shared pages. */
3081 uint32_t cZeroPages; /**< The number of zero backed pages. */
3082 uint32_t cPureMmioPages; /**< The number of pure MMIO pages. */
3083 uint32_t cMonitoredPages; /**< The number of write monitored pages. */
3084 uint32_t cWrittenToPages; /**< The number of previously write monitored pages. */
3085 uint32_t cWriteLockedPages; /**< The number of write locked pages. */
3086 uint32_t cReadLockedPages; /**< The number of read locked pages. */
3087 uint32_t cBalloonedPages; /**< The number of ballooned pages. */
3088 uint32_t cMappedChunks; /**< Number of times we mapped a chunk. */
3089 uint32_t cUnmappedChunks; /**< Number of times we unmapped a chunk. */
3090 uint32_t cLargePages; /**< The number of large pages. */
3091 uint32_t cLargePagesDisabled; /**< The number of disabled large pages. */
3092/* uint32_t aAlignment4[1]; */
3093
3094 STAMPROFILE StatLargePageAlloc; /**< Time spent by the host OS for large page allocation. */
3095 STAMCOUNTER StatLargePageAllocFailed; /**< Count allocation failures. */
3096 STAMCOUNTER StatLargePageOverflow; /**< The number of times allocating a large pages takes more than the allowed period. */
3097 STAMCOUNTER StatLargePageReused; /**< The number of large pages we've reused.*/
3098 STAMCOUNTER StatLargePageRefused; /**< The number of times we couldn't use a large page.*/
3099 STAMCOUNTER StatLargePageRecheck; /**< The number of times we rechecked a disabled large page.*/
3100 STAMCOUNTER StatLargePageTlbFlush; /**< The number of a full VCPU TLB flush was required after allocation. */
3101 STAMCOUNTER StatLargePageZeroEvict; /**< The number of zero page mappings we had to evict when allocating a large page. */
3102
3103 STAMPROFILE StatShModCheck; /**< Profiles shared module checks. */
3104
3105 STAMPROFILE StatMmio2QueryAndResetDirtyBitmap; /**< Profiling PGMR3PhysMmio2QueryAndResetDirtyBitmap. */
3106 /** @} */
3107
3108#ifdef VBOX_WITH_STATISTICS
3109 /** These are optional statistics that used to be on the hyper heap. */
3110 PGMSTATS Stats;
3111#endif
3112} PGM;
3113#ifndef IN_TSTVMSTRUCTGC /* HACK */
3114AssertCompileMemberAlignment(PGM, CritSectX, 8);
3115AssertCompileMemberAlignment(PGM, ChunkR3Map, 16);
3116AssertCompileMemberAlignment(PGM, PhysTlbR3, 32); /** @todo 32 byte alignment! */
3117AssertCompileMemberAlignment(PGM, PhysTlbR0, 32);
3118AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
3119AssertCompileMemberAlignment(PGM, aHandyPages, 8);
3120#endif /* !IN_TSTVMSTRUCTGC */
3121/** Pointer to the PGM instance data. */
3122typedef PGM *PPGM;
3123
3124
3125#ifdef VBOX_WITH_STATISTICS
3126/**
3127 * Per CPU statistis for PGM (used to be on the heap).
3128 */
3129typedef struct PGMCPUSTATS
3130{
3131 /* Common */
3132 STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /**< SyncPT - PD distribution. */
3133 STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /**< SyncPage - PD distribution. */
3134
3135 /* R0 only: */
3136 STAMPROFILE StatR0NpMiscfg; /**< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. */
3137 STAMCOUNTER StatR0NpMiscfgSyncPage; /**< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). */
3138
3139 /* RZ only: */
3140 STAMPROFILE StatRZTrap0e; /**< RC/R0: PGMTrap0eHandler() profiling. */
3141 STAMPROFILE StatRZTrap0eTime2Ballooned; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. */
3142 STAMPROFILE StatRZTrap0eTime2CSAM; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. */
3143 STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. */
3144 STAMPROFILE StatRZTrap0eTime2GuestTrap; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. */
3145 STAMPROFILE StatRZTrap0eTime2HndPhys; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. */
3146 STAMPROFILE StatRZTrap0eTime2HndUnhandled; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. */
3147 STAMPROFILE StatRZTrap0eTime2InvalidPhys; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. */
3148 STAMPROFILE StatRZTrap0eTime2MakeWritable; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. */
3149 STAMPROFILE StatRZTrap0eTime2Misc; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. */
3150 STAMPROFILE StatRZTrap0eTime2OutOfSync; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. */
3151 STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. */
3152 STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. */
3153 STAMPROFILE StatRZTrap0eTime2SyncPT; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. */
3154 STAMPROFILE StatRZTrap0eTime2WPEmulation; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. */
3155 STAMPROFILE StatRZTrap0eTime2Wp0RoUsHack; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be enabled. */
3156 STAMPROFILE StatRZTrap0eTime2Wp0RoUsUnhack; /**< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP and netware hack to be disabled. */
3157 STAMCOUNTER StatRZTrap0eConflicts; /**< RC/R0: The number of times \#PF was caused by an undetected conflict. */
3158 STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /**< RC/R0: Number of out-of-sync handled pages. */
3159 STAMCOUNTER StatRZTrap0eHandlersPhysAll; /**< RC/R0: Number of traps due to physical all-access handlers. */
3160 STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt; /**< RC/R0: Number of the physical all-access handler traps using the optimization. */
3161 STAMCOUNTER StatRZTrap0eHandlersPhysWrite; /**< RC/R0: Number of traps due to write-physical access handlers. */
3162 STAMCOUNTER StatRZTrap0eHandlersUnhandled; /**< RC/R0: Number of traps due to access outside range of monitored page(s). */
3163 STAMCOUNTER StatRZTrap0eHandlersInvalid; /**< RC/R0: Number of traps due to access to invalid physical memory. */
3164 STAMCOUNTER StatRZTrap0eUSNotPresentRead; /**< RC/R0: \#PF err kind */
3165 STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /**< RC/R0: \#PF err kind */
3166 STAMCOUNTER StatRZTrap0eUSWrite; /**< RC/R0: \#PF err kind */
3167 STAMCOUNTER StatRZTrap0eUSReserved; /**< RC/R0: \#PF err kind */
3168 STAMCOUNTER StatRZTrap0eUSNXE; /**< RC/R0: \#PF err kind */
3169 STAMCOUNTER StatRZTrap0eUSRead; /**< RC/R0: \#PF err kind */
3170 STAMCOUNTER StatRZTrap0eSVNotPresentRead; /**< RC/R0: \#PF err kind */
3171 STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /**< RC/R0: \#PF err kind */
3172 STAMCOUNTER StatRZTrap0eSVWrite; /**< RC/R0: \#PF err kind */
3173 STAMCOUNTER StatRZTrap0eSVReserved; /**< RC/R0: \#PF err kind */
3174 STAMCOUNTER StatRZTrap0eSNXE; /**< RC/R0: \#PF err kind */
3175 STAMCOUNTER StatRZTrap0eGuestPF; /**< RC/R0: Real guest \#PFs. */
3176 STAMCOUNTER StatRZTrap0eWPEmulInRZ; /**< RC/R0: WP=0 virtualization trap, handled. */
3177 STAMCOUNTER StatRZTrap0eWPEmulToR3; /**< RC/R0: WP=0 virtualization trap, chickened out. */
3178 STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /**< RC/R0: PD distribution of the \#PFs. */
3179 STAMCOUNTER StatRZGuestCR3WriteHandled; /**< RC/R0: The number of times WriteHandlerCR3() was successfully called. */
3180 STAMCOUNTER StatRZGuestCR3WriteUnhandled; /**< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. */
3181 STAMCOUNTER StatRZGuestCR3WriteConflict; /**< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. */
3182 STAMCOUNTER StatRZGuestROMWriteHandled; /**< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. */
3183 STAMCOUNTER StatRZGuestROMWriteUnhandled; /**< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler */
3184 STAMCOUNTER StatRZDynMapMigrateInvlPg; /**< RZ: invlpg in PGMR0DynMapMigrateAutoSet. */
3185 STAMPROFILE StatRZDynMapGCPageInl; /**< RZ: Calls to pgmRZDynMapGCPageInlined. */
3186 STAMCOUNTER StatRZDynMapGCPageInlHits; /**< RZ: Hash table lookup hits. */
3187 STAMCOUNTER StatRZDynMapGCPageInlMisses; /**< RZ: Misses that falls back to the code common. */
3188 STAMCOUNTER StatRZDynMapGCPageInlRamHits; /**< RZ: 1st ram range hits. */
3189 STAMCOUNTER StatRZDynMapGCPageInlRamMisses; /**< RZ: 1st ram range misses, takes slow path. */
3190 STAMPROFILE StatRZDynMapHCPageInl; /**< RZ: Calls to pgmRZDynMapHCPageInlined. */
3191 STAMCOUNTER StatRZDynMapHCPageInlHits; /**< RZ: Hash table lookup hits. */
3192 STAMCOUNTER StatRZDynMapHCPageInlMisses; /**< RZ: Misses that falls back to the code common. */
3193 STAMPROFILE StatRZDynMapHCPage; /**< RZ: Calls to pgmRZDynMapHCPageCommon. */
3194 STAMCOUNTER StatRZDynMapSetOptimize; /**< RZ: Calls to pgmRZDynMapOptimizeAutoSet. */
3195 STAMCOUNTER StatRZDynMapSetSearchFlushes; /**< RZ: Set search restoring to subset flushes. */
3196 STAMCOUNTER StatRZDynMapSetSearchHits; /**< RZ: Set search hits. */
3197 STAMCOUNTER StatRZDynMapSetSearchMisses; /**< RZ: Set search misses. */
3198 STAMCOUNTER StatRZDynMapPage; /**< RZ: Calls to pgmR0DynMapPage. */
3199 STAMCOUNTER StatRZDynMapPageHits0; /**< RZ: Hits at iPage+0. */
3200 STAMCOUNTER StatRZDynMapPageHits1; /**< RZ: Hits at iPage+1. */
3201 STAMCOUNTER StatRZDynMapPageHits2; /**< RZ: Hits at iPage+2. */
3202 STAMCOUNTER StatRZDynMapPageInvlPg; /**< RZ: invlpg. */
3203 STAMCOUNTER StatRZDynMapPageSlow; /**< RZ: Calls to pgmR0DynMapPageSlow. */
3204 STAMCOUNTER StatRZDynMapPageSlowLoopHits; /**< RZ: Hits in the pgmR0DynMapPageSlow search loop. */
3205 STAMCOUNTER StatRZDynMapPageSlowLoopMisses; /**< RZ: Misses in the pgmR0DynMapPageSlow search loop. */
3206 //STAMCOUNTER StatRZDynMapPageSlowLostHits; /**< RZ: Lost hits. */
3207 STAMCOUNTER StatRZDynMapSubsets; /**< RZ: Times PGMDynMapPushAutoSubset was called. */
3208 STAMCOUNTER StatRZDynMapPopFlushes; /**< RZ: Times PGMDynMapPopAutoSubset flushes the subset. */
3209 STAMCOUNTER aStatRZDynMapSetFilledPct[11]; /**< RZ: Set fill distribution, percent. */
3210
3211 /* HC - R3 and (maybe) R0: */
3212
3213 /* RZ & R3: */
3214 STAMPROFILE StatRZSyncCR3; /**< RC/R0: PGMSyncCR3() profiling. */
3215 STAMPROFILE StatRZSyncCR3Handlers; /**< RC/R0: Profiling of the PGMSyncCR3() update handler section. */
3216 STAMCOUNTER StatRZSyncCR3Global; /**< RC/R0: The number of global CR3 syncs. */
3217 STAMCOUNTER StatRZSyncCR3NotGlobal; /**< RC/R0: The number of non-global CR3 syncs. */
3218 STAMCOUNTER StatRZSyncCR3DstCacheHit; /**< RC/R0: The number of times we got some kind of cache hit on a page table. */
3219 STAMCOUNTER StatRZSyncCR3DstFreed; /**< RC/R0: The number of times we've had to free a shadow entry. */
3220 STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /**< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. */
3221 STAMCOUNTER StatRZSyncCR3DstNotPresent; /**< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. */
3222 STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /**< RC/R0: The number of times a global page directory wasn't flushed. */
3223 STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /**< RC/R0: The number of times a page table with only global entries wasn't flushed. */
3224 STAMPROFILE StatRZSyncPT; /**< RC/R0: PGMSyncPT() profiling. */
3225 STAMCOUNTER StatRZSyncPTFailed; /**< RC/R0: The number of times PGMSyncPT() failed. */
3226 STAMCOUNTER StatRZSyncPT4K; /**< RC/R0: Number of 4KB syncs. */
3227 STAMCOUNTER StatRZSyncPT4M; /**< RC/R0: Number of 4MB syncs. */
3228 STAMCOUNTER StatRZSyncPagePDNAs; /**< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
3229 STAMCOUNTER StatRZSyncPagePDOutOfSync; /**< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. */
3230 STAMCOUNTER StatRZAccessedPage; /**< RC/R0: The number of pages marked not present for accessed bit emulation. */
3231 STAMPROFILE StatRZDirtyBitTracking; /**< RC/R0: Profiling the dirty bit tracking in CheckPageFault(). */
3232 STAMCOUNTER StatRZDirtyPage; /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
3233 STAMCOUNTER StatRZDirtyPageBig; /**< RC/R0: The number of pages marked read-only for dirty bit tracking. */
3234 STAMCOUNTER StatRZDirtyPageSkipped; /**< RC/R0: The number of pages already dirty or readonly. */
3235 STAMCOUNTER StatRZDirtyPageTrap; /**< RC/R0: The number of traps generated for dirty bit tracking. */
3236 STAMCOUNTER StatRZDirtyPageStale; /**< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) */
3237 STAMCOUNTER StatRZDirtyTrackRealPF; /**< RC/R0: The number of real pages faults during dirty bit tracking. */
3238 STAMCOUNTER StatRZDirtiedPage; /**< RC/R0: The number of pages marked dirty because of write accesses. */
3239 STAMCOUNTER StatRZPageAlreadyDirty; /**< RC/R0: The number of pages already marked dirty because of write accesses. */
3240 STAMPROFILE StatRZInvalidatePage; /**< RC/R0: PGMInvalidatePage() profiling. */
3241 STAMCOUNTER StatRZInvalidatePage4KBPages; /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. */
3242 STAMCOUNTER StatRZInvalidatePage4MBPages; /**< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. */
3243 STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /**< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. */
3244 STAMCOUNTER StatRZInvalidatePagePDNAs; /**< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
3245 STAMCOUNTER StatRZInvalidatePagePDNPs; /**< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. */
3246 STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /**< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
3247 STAMCOUNTER StatRZInvalidatePageSizeChanges ; /**< RC/R0: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */
3248 STAMCOUNTER StatRZInvalidatePageSkipped; /**< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
3249 STAMCOUNTER StatRZPageOutOfSyncUser; /**< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
3250 STAMCOUNTER StatRZPageOutOfSyncSupervisor; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
3251 STAMCOUNTER StatRZPageOutOfSyncUserWrite; /**< RC/R0: The number of times user page is out of sync was detected in \#PF. */
3252 STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /**< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. */
3253 STAMCOUNTER StatRZPageOutOfSyncBallloon; /**< RC/R0: The number of times a ballooned page was accessed (read). */
3254 STAMPROFILE StatRZPrefetch; /**< RC/R0: PGMPrefetchPage. */
3255 STAMPROFILE StatRZFlushTLB; /**< RC/R0: Profiling of the PGMFlushTLB() body. */
3256 STAMCOUNTER StatRZFlushTLBNewCR3; /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
3257 STAMCOUNTER StatRZFlushTLBNewCR3Global; /**< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
3258 STAMCOUNTER StatRZFlushTLBSameCR3; /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
3259 STAMCOUNTER StatRZFlushTLBSameCR3Global; /**< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
3260 STAMPROFILE StatRZGstModifyPage; /**< RC/R0: Profiling of the PGMGstModifyPage() body */
3261
3262 STAMPROFILE StatR3SyncCR3; /**< R3: PGMSyncCR3() profiling. */
3263 STAMPROFILE StatR3SyncCR3Handlers; /**< R3: Profiling of the PGMSyncCR3() update handler section. */
3264 STAMCOUNTER StatR3SyncCR3Global; /**< R3: The number of global CR3 syncs. */
3265 STAMCOUNTER StatR3SyncCR3NotGlobal; /**< R3: The number of non-global CR3 syncs. */
3266 STAMCOUNTER StatR3SyncCR3DstFreed; /**< R3: The number of times we've had to free a shadow entry. */
3267 STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /**< R3: The number of times we've had to free a shadow entry for which the source entry was not present. */
3268 STAMCOUNTER StatR3SyncCR3DstNotPresent; /**< R3: The number of times we've encountered a not present shadow entry for a present guest entry. */
3269 STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /**< R3: The number of times a global page directory wasn't flushed. */
3270 STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /**< R3: The number of times a page table with only global entries wasn't flushed. */
3271 STAMCOUNTER StatR3SyncCR3DstCacheHit; /**< R3: The number of times we got some kind of cache hit on a page table. */
3272 STAMPROFILE StatR3SyncPT; /**< R3: PGMSyncPT() profiling. */
3273 STAMCOUNTER StatR3SyncPTFailed; /**< R3: The number of times PGMSyncPT() failed. */
3274 STAMCOUNTER StatR3SyncPT4K; /**< R3: Number of 4KB syncs. */
3275 STAMCOUNTER StatR3SyncPT4M; /**< R3: Number of 4MB syncs. */
3276 STAMCOUNTER StatR3SyncPagePDNAs; /**< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. */
3277 STAMCOUNTER StatR3SyncPagePDOutOfSync; /**< R3: The number of time we've encountered an out-of-sync PD in SyncPage. */
3278 STAMCOUNTER StatR3AccessedPage; /**< R3: The number of pages marked not present for accessed bit emulation. */
3279 STAMPROFILE StatR3DirtyBitTracking; /**< R3: Profiling the dirty bit tracking in CheckPageFault(). */
3280 STAMCOUNTER StatR3DirtyPage; /**< R3: The number of pages marked read-only for dirty bit tracking. */
3281 STAMCOUNTER StatR3DirtyPageBig; /**< R3: The number of pages marked read-only for dirty bit tracking. */
3282 STAMCOUNTER StatR3DirtyPageSkipped; /**< R3: The number of pages already dirty or readonly. */
3283 STAMCOUNTER StatR3DirtyPageTrap; /**< R3: The number of traps generated for dirty bit tracking. */
3284 STAMCOUNTER StatR3DirtyTrackRealPF; /**< R3: The number of real pages faults during dirty bit tracking. */
3285 STAMCOUNTER StatR3DirtiedPage; /**< R3: The number of pages marked dirty because of write accesses. */
3286 STAMCOUNTER StatR3PageAlreadyDirty; /**< R3: The number of pages already marked dirty because of write accesses. */
3287 STAMPROFILE StatR3InvalidatePage; /**< R3: PGMInvalidatePage() profiling. */
3288 STAMCOUNTER StatR3InvalidatePage4KBPages; /**< R3: The number of times PGMInvalidatePage() was called for a 4KB page. */
3289 STAMCOUNTER StatR3InvalidatePage4MBPages; /**< R3: The number of times PGMInvalidatePage() was called for a 4MB page. */
3290 STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /**< R3: The number of times PGMInvalidatePage() skipped a 4MB page. */
3291 STAMCOUNTER StatR3InvalidatePagePDNAs; /**< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. */
3292 STAMCOUNTER StatR3InvalidatePagePDNPs; /**< R3: The number of times PGMInvalidatePage() was called for a not present page directory. */
3293 STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /**< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. */
3294 STAMCOUNTER StatR3InvalidatePageSizeChanges ; /**< R3: The number of times PGMInvalidatePage() was called on a page size change (4KB <-> 2/4MB). */
3295 STAMCOUNTER StatR3InvalidatePageSkipped; /**< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. */
3296 STAMCOUNTER StatR3PageOutOfSyncUser; /**< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. */
3297 STAMCOUNTER StatR3PageOutOfSyncSupervisor; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. */
3298 STAMCOUNTER StatR3PageOutOfSyncUserWrite; /**< R3: The number of times user page is out of sync was detected in \#PF. */
3299 STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /**< R3: The number of times supervisor page is out of sync was detected in in \#PF. */
3300 STAMCOUNTER StatR3PageOutOfSyncBallloon; /**< R3: The number of times a ballooned page was accessed (read). */
3301 STAMPROFILE StatR3Prefetch; /**< R3: PGMPrefetchPage. */
3302 STAMPROFILE StatR3FlushTLB; /**< R3: Profiling of the PGMFlushTLB() body. */
3303 STAMCOUNTER StatR3FlushTLBNewCR3; /**< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) */
3304 STAMCOUNTER StatR3FlushTLBNewCR3Global; /**< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) */
3305 STAMCOUNTER StatR3FlushTLBSameCR3; /**< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) */
3306 STAMCOUNTER StatR3FlushTLBSameCR3Global; /**< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) */
3307 STAMPROFILE StatR3GstModifyPage; /**< R3: Profiling of the PGMGstModifyPage() body */
3308} PGMCPUSTATS;
3309#endif /* VBOX_WITH_STATISTICS */
3310
3311
3312/**
3313 * PGMCPU Data (part of VMCPU).
3314 */
3315typedef struct PGMCPU
3316{
3317 /** A20 gate mask.
3318 * Our current approach to A20 emulation is to let REM do it and don't bother
3319 * anywhere else. The interesting Guests will be operating with it enabled anyway.
3320 * But whould need arrise, we'll subject physical addresses to this mask. */
3321 RTGCPHYS GCPhysA20Mask;
3322 /** A20 gate state - boolean! */
3323 bool fA20Enabled;
3324 /** Mirror of the EFER.NXE bit. Managed by PGMNotifyNxeChanged. */
3325 bool fNoExecuteEnabled;
3326 /** Whether the guest CR3 and PAE PDPEs have been mapped when guest PAE mode is
3327 * active. */
3328 bool fPaePdpesAndCr3MappedR3;
3329 bool fPaePdpesAndCr3MappedR0;
3330
3331 /** What needs syncing (PGM_SYNC_*).
3332 * This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
3333 * PGMFlushTLB, and PGMR3Load. */
3334 uint32_t fSyncFlags;
3335
3336 /** The shadow paging mode. */
3337 PGMMODE enmShadowMode;
3338 /** The guest paging mode. */
3339 PGMMODE enmGuestMode;
3340 /** The guest second level address translation mode. */
3341 PGMSLAT enmGuestSlatMode;
3342 /** Guest mode data table index (PGM_TYPE_XXX). */
3343 uint8_t volatile idxGuestModeData;
3344 /** Shadow mode data table index (PGM_TYPE_XXX). */
3345 uint8_t volatile idxShadowModeData;
3346 /** Both mode data table index (complicated). */
3347 uint8_t volatile idxBothModeData;
3348 /** Alignment padding. */
3349 uint8_t abPadding[1];
3350
3351 /** The guest CR3.
3352 * When SLAT is active, this is the translated physical address.
3353 * When SLAT is inactive, this is the physical address in CR3. */
3354 RTGCPHYS GCPhysCR3;
3355
3356 /** The nested-guest CR3.
3357 * When SLAT is active, this is CR3 prior to translation.
3358 * When SLAT is inactive, this is unused (and NIL_RTGCPHYS). */
3359 RTGCPHYS GCPhysNstGstCR3;
3360
3361 /** @name 32-bit Guest Paging.
3362 * @{ */
3363 /** The guest's page directory, R3 pointer. */
3364 R3PTRTYPE(PX86PD) pGst32BitPdR3;
3365 /** The guest's page directory, R0 pointer. */
3366 R0PTRTYPE(PX86PD) pGst32BitPdR0;
3367 /** Mask containing the MBZ bits of a big page PDE. */
3368 uint32_t fGst32BitMbzBigPdeMask;
3369 /** Set if the page size extension (PSE) is enabled. */
3370 bool fGst32BitPageSizeExtension;
3371 /** Alignment padding. */
3372 bool afAlignment2[3];
3373 /** @} */
3374
3375 /** @name PAE Guest Paging.
3376 * @{ */
3377 /** The guest's page directory pointer table, R3 pointer. */
3378 R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
3379 /** The guest's page directory pointer table, R0 pointer. */
3380 R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
3381
3382 /** The guest's page directories, R3 pointers.
3383 * These are individual pointers and don't have to be adjacent.
3384 * These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
3385 R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
3386 /** The guest's page directories, R0 pointers.
3387 * Same restrictions as apGstPaePDsR3. */
3388 R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
3389 /** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
3390 RTGCPHYS aGCPhysGstPaePDs[4];
3391 /** The physical addresses of the monitored guest page directories (PAE). */
3392 RTGCPHYS aGCPhysGstPaePDsMonitored[4];
3393 /** Mask containing the MBZ PTE bits. */
3394 uint64_t fGstPaeMbzPteMask;
3395 /** Mask containing the MBZ PDE bits. */
3396 uint64_t fGstPaeMbzPdeMask;
3397 /** Mask containing the MBZ big page PDE bits. */
3398 uint64_t fGstPaeMbzBigPdeMask;
3399 /** Mask containing the MBZ PDPE bits. */
3400 uint64_t fGstPaeMbzPdpeMask;
3401 /** @} */
3402
3403 /** @name AMD64 Guest Paging.
3404 * @{ */
3405 /** The guest's page directory pointer table, R3 pointer. */
3406 R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
3407 /** The guest's page directory pointer table, R0 pointer. */
3408 R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
3409 /** Mask containing the MBZ PTE bits. */
3410 uint64_t fGstAmd64MbzPteMask;
3411 /** Mask containing the MBZ PDE bits. */
3412 uint64_t fGstAmd64MbzPdeMask;
3413 /** Mask containing the MBZ big page PDE bits. */
3414 uint64_t fGstAmd64MbzBigPdeMask;
3415 /** Mask containing the MBZ PDPE bits. */
3416 uint64_t fGstAmd64MbzPdpeMask;
3417 /** Mask containing the MBZ big page PDPE bits. */
3418 uint64_t fGstAmd64MbzBigPdpeMask;
3419 /** Mask containing the MBZ PML4E bits. */
3420 uint64_t fGstAmd64MbzPml4eMask;
3421 /** Mask containing the PDPE bits that we shadow. */
3422 uint64_t fGstAmd64ShadowedPdpeMask;
3423 /** Mask containing the PML4E bits that we shadow. */
3424 uint64_t fGstAmd64ShadowedPml4eMask;
3425 /** @} */
3426
3427 /** @name PAE and AMD64 Guest Paging.
3428 * @{ */
3429 /** Mask containing the PTE bits that we shadow. */
3430 uint64_t fGst64ShadowedPteMask;
3431 /** Mask containing the PDE bits that we shadow. */
3432 uint64_t fGst64ShadowedPdeMask;
3433 /** Mask containing the big page PDE bits that we shadow in the PDE. */
3434 uint64_t fGst64ShadowedBigPdeMask;
3435 /** Mask containing the big page PDE bits that we shadow in the PTE. */
3436 uint64_t fGst64ShadowedBigPde4PteMask;
3437 /** @} */
3438
3439 /** @name EPT Guest Paging.
3440 * @{ */
3441 /** The guest's page directory pointer table, R3 pointer. */
3442 R3PTRTYPE(PEPTPML4) pGstEptPml4R3;
3443 /** The guest's page directory pointer table, R0 pointer. */
3444 R0PTRTYPE(PEPTPML4) pGstEptPml4R0;
3445 /** The guest's EPT pointer (copy of virtual VMCS). */
3446 uint64_t uEptPtr;
3447 /** Copy of the VM's IA32_VMX_EPT_VPID_CAP VPID MSR for faster access. Doesn't
3448 * change through the lifetime of the VM. */
3449 uint64_t uEptVpidCapMsr;
3450 /** Mask containing the MBZ PTE bits. */
3451 uint64_t fGstEptMbzPteMask;
3452 /** Mask containing the MBZ PDE bits. */
3453 uint64_t fGstEptMbzPdeMask;
3454 /** Mask containing the MBZ big page (2M) PDE bits. */
3455 uint64_t fGstEptMbzBigPdeMask;
3456 /** Mask containing the MBZ PDPTE bits. */
3457 uint64_t fGstEptMbzPdpteMask;
3458 /** Mask containing the MBZ big page (1G) PDPTE bits. */
3459 uint64_t fGstEptMbzBigPdpteMask;
3460 /** Mask containing the MBZ PML4E bits. */
3461 uint64_t fGstEptMbzPml4eMask;
3462 /** Mask to determine whether an entry is present. */
3463 uint64_t fGstEptPresentMask;
3464 /** @} */
3465
3466 /** Pointer to the page of the current active CR3 - R3 Ptr. */
3467 R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
3468 /** Pointer to the page of the current active CR3 - R0 Ptr. */
3469 R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
3470
3471 /** For saving stack space, the disassembler state is allocated here instead of
3472 * on the stack. */
3473 DISCPUSTATE DisState;
3474
3475 /** Counts the number of times the netware WP0+RO+US hack has been applied. */
3476 uint64_t cNetwareWp0Hacks;
3477
3478 /** Count the number of pgm pool access handler calls. */
3479 uint64_t cPoolAccessHandler;
3480
3481 /** @name Release Statistics
3482 * @{ */
3483 /** The number of times the guest has switched mode since last reset or statistics reset. */
3484 STAMCOUNTER cGuestModeChanges;
3485 /** The number of times the guest has switched mode since last reset or statistics reset. */
3486 STAMCOUNTER cA20Changes;
3487 /** @} */
3488
3489#ifdef VBOX_WITH_STATISTICS
3490 /** These are statistics that used to be on the hyper heap. */
3491 PGMCPUSTATS Stats;
3492#endif
3493} PGMCPU;
3494/** Pointer to the per-cpu PGM data. */
3495typedef PGMCPU *PPGMCPU;
3496
3497
3498/** @name PGM::fSyncFlags Flags
3499 * @note Was part of saved state a long time ago.
3500 * @{
3501 */
3502/* 0 used to be PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL */
3503/** Always sync CR3. */
3504#define PGM_SYNC_ALWAYS RT_BIT(1)
3505/** Check guest mapping in SyncCR3. */
3506#define PGM_SYNC_MAP_CR3 RT_BIT(3)
3507/** Clear the page pool (a light weight flush). */
3508#define PGM_SYNC_CLEAR_PGM_POOL_BIT 8
3509#define PGM_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
3510/** @} */
3511
3512
3513/**
3514 * PGM GVMCPU instance data.
3515 */
3516typedef struct PGMR0PERVCPU
3517{
3518#ifdef VBOX_WITH_STATISTICS
3519 /** R0: Which statistic this \#PF should be attributed to. */
3520 R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
3521#endif
3522 uint64_t u64Dummy;
3523} PGMR0PERVCPU;
3524
3525
3526/**
3527 * PGM GVM instance data.
3528 */
3529typedef struct PGMR0PERVM
3530{
3531 /** @name PGM Pool related stuff.
3532 * @{ */
3533 /** Critical section for serializing pool growth. */
3534 RTCRITSECT PoolGrowCritSect;
3535 /** The memory objects for the pool pages. */
3536 RTR0MEMOBJ ahPoolMemObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW];
3537 /** The ring-3 mapping objects for the pool pages. */
3538 RTR0MEMOBJ ahPoolMapObjs[(PGMPOOL_IDX_LAST + PGMPOOL_CFG_MAX_GROW - 1) / PGMPOOL_CFG_MAX_GROW];
3539 /** @} */
3540} PGMR0PERVM;
3541
3542RT_C_DECLS_BEGIN
3543
3544#if defined(VBOX_STRICT)
3545int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL);
3546# define PGM_LOCK_VOID(a_pVM) pgmLockDebug((a_pVM), true, RT_SRC_POS)
3547# define PGM_LOCK(a_pVM) pgmLockDebug((a_pVM), false, RT_SRC_POS)
3548#else
3549int pgmLock(PVMCC pVM, bool fVoid);
3550# define PGM_LOCK_VOID(a_pVM) pgmLock((a_pVM), true)
3551# define PGM_LOCK(a_pVM) pgmLock((a_pVM), false)
3552#endif
3553void pgmUnlock(PVMCC pVM);
3554# define PGM_UNLOCK(a_pVM) pgmUnlock((a_pVM))
3555/**
3556 * Asserts that the caller owns the PDM lock.
3557 * This is the internal variant of PGMIsLockOwner.
3558 * @param a_pVM Pointer to the VM.
3559 */
3560#define PGM_LOCK_ASSERT_OWNER(a_pVM) Assert(PDMCritSectIsOwner((a_pVM), &(a_pVM)->pgm.s.CritSectX))
3561/**
3562 * Asserts that the caller owns the PDM lock.
3563 * This is the internal variant of PGMIsLockOwner.
3564 * @param a_pVM Pointer to the VM.
3565 * @param a_pVCpu The current CPU handle.
3566 */
3567#define PGM_LOCK_ASSERT_OWNER_EX(a_pVM, a_pVCpu) Assert(PDMCritSectIsOwnerEx((a_pVCpu), &(a_pVM)->pgm.s.CritSectX))
3568
3569int pgmHandlerPhysicalExCreate(PVMCC pVM, PGMPHYSHANDLERTYPE hType, RTR3PTR pvUserR3, RTR0PTR pvUserR0,
3570 RTRCPTR pvUserRC, R3PTRTYPE(const char *) pszDesc, PPGMPHYSHANDLER *ppPhysHandler);
3571int pgmHandlerPhysicalExDup(PVMCC pVM, PPGMPHYSHANDLER pPhysHandlerSrc, PPGMPHYSHANDLER *ppPhysHandler);
3572int pgmHandlerPhysicalExRegister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast);
3573int pgmHandlerPhysicalExDeregister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler);
3574int pgmHandlerPhysicalExDestroy(PVMCC pVM, PPGMPHYSHANDLER pHandler);
3575void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
3576bool pgmHandlerPhysicalIsAll(PVMCC pVM, RTGCPHYS GCPhys);
3577void pgmHandlerPhysicalResetAliasedPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, PPGMRAMRANGE pRam, bool fDoAccounting);
3578DECLHIDDEN(int) pgmHandlerPhysicalResetMmio2WithBitmap(PVMCC pVM, RTGCPHYS GCPhys, void *pvBitmap, uint32_t offBitmap);
3579DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3580int pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
3581
3582int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3583int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys);
3584#ifdef IN_RING0
3585int pgmR0PhysAllocateHandyPages(PGVM pGVM, VMCPUID idCpu, bool fRing3);
3586int pgmR0PhysAllocateLargePage(PGVM pGVM, VMCPUID idCpu, RTGCPHYS GCPhys);
3587#endif
3588int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
3589int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys);
3590int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3591void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3592int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3593int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3594int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3595int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
3596int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
3597int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
3598int pgmPhysCr3ToHCPtr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
3599int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3600int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock);
3601int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock);
3602void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock);
3603PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) pgmPhysRomWriteHandler;
3604PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) pgmPhysMmio2WriteHandler;
3605#ifndef IN_RING3
3606DECLEXPORT(FNPGMPHYSHANDLER) pgmPhysHandlerRedirectToHC;
3607DECLEXPORT(FNPGMRZPHYSPFHANDLER) pgmPhysPfHandlerRedirectToHC;
3608DECLEXPORT(FNPGMRZPHYSPFHANDLER) pgmPhysRomWritePfHandler;
3609DECLEXPORT(FNPGMRZPHYSPFHANDLER) pgmPhysMmio2WritePfHandler;
3610#endif
3611int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
3612 PGMPAGETYPE enmNewType);
3613void pgmPhysInvalidRamRangeTlbs(PVMCC pVM);
3614void pgmPhysInvalidatePageMapTLB(PVMCC pVM);
3615void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys);
3616PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys);
3617PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys);
3618PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys);
3619int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage);
3620int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam);
3621#ifdef VBOX_WITH_NATIVE_NEM
3622void pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State);
3623#endif
3624
3625#ifdef IN_RING3
3626void pgmR3PhysRelinkRamRanges(PVM pVM);
3627int pgmR3PhysRamPreAllocate(PVM pVM);
3628int pgmR3PhysRamReset(PVM pVM);
3629int pgmR3PhysRomReset(PVM pVM);
3630int pgmR3PhysRamZeroAll(PVM pVM);
3631int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
3632int pgmR3PhysRamTerm(PVM pVM);
3633void pgmR3PhysRomTerm(PVM pVM);
3634void pgmR3PhysAssertSharedPageChecksums(PVM pVM);
3635
3636int pgmR3PoolInit(PVM pVM);
3637void pgmR3PoolRelocate(PVM pVM);
3638void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu);
3639void pgmR3PoolReset(PVM pVM);
3640void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb);
3641DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl);
3642void pgmR3PoolWriteProtectPages(PVM pVM);
3643
3644#endif /* IN_RING3 */
3645int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
3646 uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage);
3647void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
3648void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
3649int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */);
3650void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
3651PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
3652PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
3653int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv);
3654int pgmPoolSyncCR3(PVMCPUCC pVCpu);
3655bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys);
3656void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT);
3657int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
3658void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte);
3659uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte);
3660void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte);
3661void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3662void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3663PGM_ALL_CB2_PROTO(FNPGMPHYSHANDLER) pgmPoolAccessHandler;
3664#ifndef IN_RING3
3665DECLEXPORT(FNPGMRZPHYSPFHANDLER) pgmRZPoolAccessPfHandler;
3666#endif
3667
3668void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3669void pgmPoolResetDirtyPages(PVMCC pVM);
3670void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage);
3671
3672int pgmR3ExitShadowModeBeforePoolFlush(PVMCPU pVCpu);
3673int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
3674void pgmR3RefreshShadowModeAfterA20Change(PVMCPU pVCpu);
3675
3676int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags);
3677int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
3678int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode);
3679
3680int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd);
3681int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt);
3682int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd);
3683int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4);
3684#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3685int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppPml4);
3686#endif
3687int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk);
3688int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk);
3689
3690# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64 && defined(IN_RING3)
3691FNDBGCCMD pgmR3CmdCheckDuplicatePages;
3692FNDBGCCMD pgmR3CmdShowSharedModules;
3693# endif
3694
3695void pgmLogState(PVM pVM);
3696
3697RT_C_DECLS_END
3698
3699/** @} */
3700
3701#endif /* !VMM_INCLUDED_SRC_include_PGMInternal_h */
3702
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette