PGMInternal.h@ 38953

Last change on this file since 38953 was 38953, checked in by vboxsync, 13 years ago
PGM: Attempt at fixing the VERR_MAP_FAILED during state save problem on 32-bit hosts when assigning lots of memory to the guest. PGM should lock down guest RAM pages before use and release them afterwards like everyone else. Still quite some stuff left to do there, so I've deviced a little hack for tracking unlocked mappings and using this as input when deciding to do async or sync chunk unmapping at save/load time. See xtracker #5912 and public ticket 7929.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 181.3 KB

Line
1	/* $Id: PGMInternal.h 38953 2011-10-06 08:49:36Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	#ifndef ___PGMInternal_h
19	#define ___PGMInternal_h
20
21	#include <VBox/cdefs.h>
22	#include <VBox/types.h>
23	#include <VBox/err.h>
24	#include <VBox/dbg.h>
25	#include <VBox/vmm/stam.h>
26	#include <VBox/param.h>
27	#include <VBox/vmm/vmm.h>
28	#include <VBox/vmm/mm.h>
29	#include <VBox/vmm/pdmcritsect.h>
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/dis.h>
32	#include <VBox/vmm/dbgf.h>
33	#include <VBox/log.h>
34	#include <VBox/vmm/gmm.h>
35	#include <VBox/vmm/hwaccm.h>
36	#include <VBox/vmm/hwacc_vmx.h>
37	#include "internal/pgm.h"
38	#include <iprt/asm.h>
39	#include <iprt/assert.h>
40	#include <iprt/avl.h>
41	#include <iprt/critsect.h>
42	#include <iprt/sha.h>
43
44
45
46	/** @defgroup grp_pgm_int Internals
47	* @ingroup grp_pgm
48	* @internal
49	* @{
50	*/
51
52
53	/** @name PGM Compile Time Config
54	* @{
55	*/
56
57	/**
58	* Indicates that there are no guest mappings to care about.
59	* Currently on raw-mode related code uses mappings, i.e. RC and R3 code.
60	*/
61	#if defined(IN_RING0) \|\| !defined(VBOX_WITH_RAW_MODE)
62	# define PGM_WITHOUT_MAPPINGS
63	#endif
64
65	/**
66	* Check and skip global PDEs for non-global flushes
67	*/
68	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
69
70	/**
71	* Optimization for PAE page tables that are modified often
72	*/
73	//#if 0 /* disabled again while debugging */
74	#ifndef IN_RC
75	# define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
76	#endif
77	//#endif
78
79	/**
80	* Large page support enabled only on 64 bits hosts; applies to nested paging only.
81	*/
82	#if (HC_ARCH_BITS == 64) && !defined(IN_RC)
83	# define PGM_WITH_LARGE_PAGES
84	#endif
85
86	/**
87	* Enables optimizations for MMIO handlers that exploits X86_TRAP_PF_RSVD and
88	* VMX_EXIT_EPT_MISCONFIG.
89	*/
90	#if 1 /* testing */
91	# define PGM_WITH_MMIO_OPTIMIZATIONS
92	#endif
93
94	/**
95	* Sync N pages instead of a whole page table
96	*/
97	#define PGM_SYNC_N_PAGES
98
99	/**
100	* Number of pages to sync during a page fault
101	*
102	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
103	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
104	*
105	* Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
106	* world switch overhead, so let's sync more.
107	*/
108	# ifdef IN_RING0
109	/* Chose 32 based on the compile test in #4219; 64 shows worse stats.
110	* 32 again shows better results than 16; slightly more overhead in the \#PF handler,
111	* but ~5% fewer faults.
112	*/
113	# define PGM_SYNC_NR_PAGES 32
114	#else
115	# define PGM_SYNC_NR_PAGES 8
116	#endif
117
118	/**
119	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
120	*/
121	#define PGM_MAX_PHYSCACHE_ENTRIES 64
122	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
123
124
125	/** @def PGMPOOL_CFG_MAX_GROW
126	* The maximum number of pages to add to the pool in one go.
127	*/
128	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
129
130	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
131	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
132	*/
133	#ifdef VBOX_STRICT
134	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
135	#endif
136
137	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
138	* Enables the experimental lazy page allocation code. */
139	/#define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
140
141	/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
142	* Enables real write monitoring of pages, i.e. mapping them read-only and
143	* only making them writable when getting a write access #PF. */
144	#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
145
146	/** @} */
147
148
149	/** @name PDPT and PML4 flags.
150	* These are placed in the three bits available for system programs in
151	* the PDPT and PML4 entries.
152	* @{ */
153	/** The entry is a permanent one and it's must always be present.
154	* Never free such an entry. */
155	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
156	/** Mapping (hypervisor allocated pagetable). */
157	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
158	/** @} */
159
160	/** @name Page directory flags.
161	* These are placed in the three bits available for system programs in
162	* the page directory entries.
163	* @{ */
164	/** Mapping (hypervisor allocated pagetable). */
165	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
166	/** Made read-only to facilitate dirty bit tracking. */
167	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
168	/** @} */
169
170	/** @name Page flags.
171	* These are placed in the three bits available for system programs in
172	* the page entries.
173	* @{ */
174	/** Made read-only to facilitate dirty bit tracking. */
175	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
176
177	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
178	/** Scanned and approved by CSAM (tm).
179	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
180	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/vmm/pgm.h. */
181	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
182	#endif
183
184	/** @} */
185
186	/** @name Defines used to indicate the shadow and guest paging in the templates.
187	* @{ */
188	#define PGM_TYPE_REAL 1
189	#define PGM_TYPE_PROT 2
190	#define PGM_TYPE_32BIT 3
191	#define PGM_TYPE_PAE 4
192	#define PGM_TYPE_AMD64 5
193	#define PGM_TYPE_NESTED 6
194	#define PGM_TYPE_EPT 7
195	#define PGM_TYPE_MAX PGM_TYPE_EPT
196	/** @} */
197
198	/** Macro for checking if the guest is using paging.
199	* @param uGstType PGM_TYPE_*
200	* @param uShwType PGM_TYPE_*
201	* @remark ASSUMES certain order of the PGM_TYPE_* values.
202	*/
203	#define PGM_WITH_PAGING(uGstType, uShwType) \
204	( (uGstType) >= PGM_TYPE_32BIT \
205	&& (uShwType) != PGM_TYPE_NESTED \
206	&& (uShwType) != PGM_TYPE_EPT)
207
208	/** Macro for checking if the guest supports the NX bit.
209	* @param uGstType PGM_TYPE_*
210	* @param uShwType PGM_TYPE_*
211	* @remark ASSUMES certain order of the PGM_TYPE_* values.
212	*/
213	#define PGM_WITH_NX(uGstType, uShwType) \
214	( (uGstType) >= PGM_TYPE_PAE \
215	&& (uShwType) != PGM_TYPE_NESTED \
216	&& (uShwType) != PGM_TYPE_EPT)
217
218
219	/** @def PGM_HCPHYS_2_PTR
220	* Maps a HC physical page pool address to a virtual address.
221	*
222	* @returns VBox status code.
223	* @param pVM The VM handle.
224	* @param pVCpu The current CPU.
225	* @param HCPhys The HC physical address to map to a virtual one.
226	* @param ppv Where to store the virtual address. No need to cast
227	* this.
228	*
229	* @remark Use with care as we don't have so much dynamic mapping space in
230	* ring-0 on 32-bit darwin and in RC.
231	* @remark There is no need to assert on the result.
232	*/
233	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
234	# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
235	pgmRZDynMapHCPageInlined(pVCpu, HCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
236	#else
237	# define PGM_HCPHYS_2_PTR(pVM, pVCpu, HCPhys, ppv) \
238	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
239	#endif
240
241	/** @def PGM_GCPHYS_2_PTR_V2
242	* Maps a GC physical page address to a virtual address.
243	*
244	* @returns VBox status code.
245	* @param pVM The VM handle.
246	* @param pVCpu The current CPU.
247	* @param GCPhys The GC physical address to map to a virtual one.
248	* @param ppv Where to store the virtual address. No need to cast this.
249	*
250	* @remark Use with care as we don't have so much dynamic mapping space in
251	* ring-0 on 32-bit darwin and in RC.
252	* @remark There is no need to assert on the result.
253	*/
254	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
255	# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
256	pgmRZDynMapGCPageV2Inlined(pVM, pVCpu, GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
257	#else
258	# define PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GCPhys, ppv) \
259	pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
260	#endif
261
262	/** @def PGM_GCPHYS_2_PTR
263	* Maps a GC physical page address to a virtual address.
264	*
265	* @returns VBox status code.
266	* @param pVM The VM handle.
267	* @param GCPhys The GC physical address to map to a virtual one.
268	* @param ppv Where to store the virtual address. No need to cast this.
269	*
270	* @remark Use with care as we don't have so much dynamic mapping space in
271	* ring-0 on 32-bit darwin and in RC.
272	* @remark There is no need to assert on the result.
273	*/
274	#define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2(pVM, VMMGetCpu(pVM), GCPhys, ppv)
275
276	/** @def PGM_GCPHYS_2_PTR_BY_VMCPU
277	* Maps a GC physical page address to a virtual address.
278	*
279	* @returns VBox status code.
280	* @param pVCpu The current CPU.
281	* @param GCPhys The GC physical address to map to a virtual one.
282	* @param ppv Where to store the virtual address. No need to cast this.
283	*
284	* @remark Use with care as we don't have so much dynamic mapping space in
285	* ring-0 on 32-bit darwin and in RC.
286	* @remark There is no need to assert on the result.
287	*/
288	#define PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhys, ppv) PGM_GCPHYS_2_PTR_V2((pVCpu)->CTX_SUFF(pVM), pVCpu, GCPhys, ppv)
289
290	/** @def PGM_GCPHYS_2_PTR_EX
291	* Maps a unaligned GC physical page address to a virtual address.
292	*
293	* @returns VBox status code.
294	* @param pVM The VM handle.
295	* @param GCPhys The GC physical address to map to a virtual one.
296	* @param ppv Where to store the virtual address. No need to cast this.
297	*
298	* @remark Use with care as we don't have so much dynamic mapping space in
299	* ring-0 on 32-bit darwin and in RC.
300	* @remark There is no need to assert on the result.
301	*/
302	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
303	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
304	pgmRZDynMapGCPageOffInlined(VMMGetCpu(pVM), GCPhys, (void **)(ppv) RTLOG_COMMA_SRC_POS)
305	#else
306	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
307	pgmPhysGCPhys2R3Ptr(pVM, GCPhys, (PRTR3PTR)(ppv)) /** @todo this isn't asserting! */
308	#endif
309
310	/** @def PGM_DYNMAP_UNUSED_HINT
311	* Hints to the dynamic mapping code in RC and R0/darwin that the specified page
312	* is no longer used.
313	*
314	* For best effect only apply this to the page that was mapped most recently.
315	*
316	* @param pVCpu The current CPU.
317	* @param pvPage The pool page.
318	*/
319	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
320	# ifdef LOG_ENABLED
321	# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) pgmRZDynMapUnusedHint(pVCpu, pvPage, RT_SRC_POS)
322	# else
323	# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) pgmRZDynMapUnusedHint(pVCpu, pvPage)
324	# endif
325	#else
326	# define PGM_DYNMAP_UNUSED_HINT(pVCpu, pvPage) do {} while (0)
327	#endif
328
329	/** @def PGM_DYNMAP_UNUSED_HINT_VM
330	* Hints to the dynamic mapping code in RC and R0/darwin that the specified page
331	* is no longer used.
332	*
333	* For best effect only apply this to the page that was mapped most recently.
334	*
335	* @param pVM The VM handle.
336	* @param pvPage The pool page.
337	*/
338	#define PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvPage) PGM_DYNMAP_UNUSED_HINT(VMMGetCpu(pVM), pvPage)
339
340
341	/** @def PGM_INVL_PG
342	* Invalidates a page.
343	*
344	* @param pVCpu The VMCPU handle.
345	* @param GCVirt The virtual address of the page to invalidate.
346	*/
347	#ifdef IN_RC
348	# define PGM_INVL_PG(pVCpu, GCVirt) ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
349	#elif defined(IN_RING0)
350	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
351	#else
352	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
353	#endif
354
355	/** @def PGM_INVL_PG_ALL_VCPU
356	* Invalidates a page on all VCPUs
357	*
358	* @param pVM The VM handle.
359	* @param GCVirt The virtual address of the page to invalidate.
360	*/
361	#ifdef IN_RC
362	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) ASMInvalidatePage((void *)(uintptr_t)(GCVirt))
363	#elif defined(IN_RING0)
364	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
365	#else
366	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
367	#endif
368
369	/** @def PGM_INVL_BIG_PG
370	* Invalidates a 4MB page directory entry.
371	*
372	* @param pVCpu The VMCPU handle.
373	* @param GCVirt The virtual address within the page directory to invalidate.
374	*/
375	#ifdef IN_RC
376	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) ASMReloadCR3()
377	#elif defined(IN_RING0)
378	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
379	#else
380	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
381	#endif
382
383	/** @def PGM_INVL_VCPU_TLBS()
384	* Invalidates the TLBs of the specified VCPU
385	*
386	* @param pVCpu The VMCPU handle.
387	*/
388	#ifdef IN_RC
389	# define PGM_INVL_VCPU_TLBS(pVCpu) ASMReloadCR3()
390	#elif defined(IN_RING0)
391	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
392	#else
393	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
394	#endif
395
396	/** @def PGM_INVL_ALL_VCPU_TLBS()
397	* Invalidates the TLBs of all VCPUs
398	*
399	* @param pVM The VM handle.
400	*/
401	#ifdef IN_RC
402	# define PGM_INVL_ALL_VCPU_TLBS(pVM) ASMReloadCR3()
403	#elif defined(IN_RING0)
404	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
405	#else
406	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
407	#endif
408
409
410	/** @name Safer Shadow PAE PT/PTE
411	* For helping avoid misinterpreting invalid PAE/AMD64 page table entries as
412	* present.
413	*
414	* @{
415	*/
416	#if 1
417	/**
418	* For making sure that u1Present and X86_PTE_P checks doesn't mistake
419	* invalid entries for present.
420	* @sa X86PTEPAE.
421	*/
422	typedef union PGMSHWPTEPAE
423	{
424	/** Unsigned integer view */
425	X86PGPAEUINT uCareful;
426	/* Not other views. */
427	} PGMSHWPTEPAE;
428
429	# define PGMSHWPTEPAE_IS_P(Pte) ( ((Pte).uCareful & (X86_PTE_P \| X86_PTE_PAE_MBZ_MASK_NX)) == X86_PTE_P )
430	# define PGMSHWPTEPAE_IS_RW(Pte) ( !!((Pte).uCareful & X86_PTE_RW))
431	# define PGMSHWPTEPAE_IS_US(Pte) ( !!((Pte).uCareful & X86_PTE_US))
432	# define PGMSHWPTEPAE_IS_A(Pte) ( !!((Pte).uCareful & X86_PTE_A))
433	# define PGMSHWPTEPAE_IS_D(Pte) ( !!((Pte).uCareful & X86_PTE_D))
434	# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).uCareful & PGM_PTFLAGS_TRACK_DIRTY) )
435	# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).uCareful & (X86_PTE_P \| X86_PTE_RW \| X86_PTE_PAE_MBZ_MASK_NX)) == (X86_PTE_P \| X86_PTE_RW) )
436	# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).uCareful )
437	# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).uCareful & X86_PTE_PAE_PG_MASK )
438	# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).uCareful ) /*< Use with care. /
439	# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).uCareful = (uVal); } while (0)
440	# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).uCareful = (Pte2).uCareful; } while (0)
441	# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).uCareful, (uVal)); } while (0)
442	# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).uCareful, (Pte2).uCareful); } while (0)
443	# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).uCareful &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
444	# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).uCareful \|= X86_PTE_RW; } while (0)
445
446	/**
447	* For making sure that u1Present and X86_PTE_P checks doesn't mistake
448	* invalid entries for present.
449	* @sa X86PTPAE.
450	*/
451	typedef struct PGMSHWPTPAE
452	{
453	PGMSHWPTEPAE a[X86_PG_PAE_ENTRIES];
454	} PGMSHWPTPAE;
455
456	#else
457	typedef X86PTEPAE PGMSHWPTEPAE;
458	typedef X86PTPAE PGMSHWPTPAE;
459	# define PGMSHWPTEPAE_IS_P(Pte) ( (Pte).n.u1Present )
460	# define PGMSHWPTEPAE_IS_RW(Pte) ( (Pte).n.u1Write )
461	# define PGMSHWPTEPAE_IS_US(Pte) ( (Pte).n.u1User )
462	# define PGMSHWPTEPAE_IS_A(Pte) ( (Pte).n.u1Accessed )
463	# define PGMSHWPTEPAE_IS_D(Pte) ( (Pte).n.u1Dirty )
464	# define PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte) ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
465	# define PGMSHWPTEPAE_IS_P_RW(Pte) ( ((Pte).u & (X86_PTE_P \| X86_PTE_RW)) == (X86_PTE_P \| X86_PTE_RW) )
466	# define PGMSHWPTEPAE_GET_LOG(Pte) ( (Pte).u )
467	# define PGMSHWPTEPAE_GET_HCPHYS(Pte) ( (Pte).u & X86_PTE_PAE_PG_MASK )
468	# define PGMSHWPTEPAE_GET_U(Pte) ( (Pte).u ) /*< Use with care. /
469	# define PGMSHWPTEPAE_SET(Pte, uVal) do { (Pte).u = (uVal); } while (0)
470	# define PGMSHWPTEPAE_SET2(Pte, Pte2) do { (Pte).u = (Pte2).u; } while (0)
471	# define PGMSHWPTEPAE_ATOMIC_SET(Pte, uVal) do { ASMAtomicWriteU64(&(Pte).u, (uVal)); } while (0)
472	# define PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
473	# define PGMSHWPTEPAE_SET_RO(Pte) do { (Pte).u &= ~(X86PGPAEUINT)X86_PTE_RW; } while (0)
474	# define PGMSHWPTEPAE_SET_RW(Pte) do { (Pte).u \|= X86_PTE_RW; } while (0)
475
476	#endif
477
478	/** Pointer to a shadow PAE PTE. */
479	typedef PGMSHWPTEPAE *PPGMSHWPTEPAE;
480	/** Pointer to a const shadow PAE PTE. */
481	typedef PGMSHWPTEPAE const *PCPGMSHWPTEPAE;
482
483	/** Pointer to a shadow PAE page table. */
484	typedef PGMSHWPTPAE *PPGMSHWPTPAE;
485	/** Pointer to a const shadow PAE page table. */
486	typedef PGMSHWPTPAE const *PCPGMSHWPTPAE;
487	/** @} */
488
489
490	/** Size of the GCPtrConflict array in PGMMAPPING.
491	* @remarks Must be a power of two. */
492	#define PGMMAPPING_CONFLICT_MAX 8
493
494	/**
495	* Structure for tracking GC Mappings.
496	*
497	* This structure is used by linked list in both GC and HC.
498	*/
499	typedef struct PGMMAPPING
500	{
501	/** Pointer to next entry. */
502	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
503	/** Pointer to next entry. */
504	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
505	/** Pointer to next entry. */
506	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
507	/** Indicate whether this entry is finalized. */
508	bool fFinalized;
509	/** Start Virtual address. */
510	RTGCPTR GCPtr;
511	/** Last Virtual address (inclusive). */
512	RTGCPTR GCPtrLast;
513	/** Range size (bytes). */
514	RTGCPTR cb;
515	/** Pointer to relocation callback function. */
516	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
517	/** User argument to the callback. */
518	R3PTRTYPE(void *) pvUser;
519	/** Mapping description / name. For easing debugging. */
520	R3PTRTYPE(const char *) pszDesc;
521	/** Last 8 addresses that caused conflicts. */
522	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
523	/** Number of conflicts for this hypervisor mapping. */
524	uint32_t cConflicts;
525	/** Number of page tables. */
526	uint32_t cPTs;
527
528	/** Array of page table mapping data. Each entry
529	* describes one page table. The array can be longer
530	* than the declared length.
531	*/
532	struct
533	{
534	/** The HC physical address of the page table. */
535	RTHCPHYS HCPhysPT;
536	/** The HC physical address of the first PAE page table. */
537	RTHCPHYS HCPhysPaePT0;
538	/** The HC physical address of the second PAE page table. */
539	RTHCPHYS HCPhysPaePT1;
540	/** The HC virtual address of the 32-bit page table. */
541	R3PTRTYPE(PX86PT) pPTR3;
542	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
543	R3PTRTYPE(PPGMSHWPTPAE) paPaePTsR3;
544	/** The RC virtual address of the 32-bit page table. */
545	RCPTRTYPE(PX86PT) pPTRC;
546	/** The RC virtual address of the two PAE page table. */
547	RCPTRTYPE(PPGMSHWPTPAE) paPaePTsRC;
548	/** The R0 virtual address of the 32-bit page table. */
549	R0PTRTYPE(PX86PT) pPTR0;
550	/** The R0 virtual address of the two PAE page table. */
551	R0PTRTYPE(PPGMSHWPTPAE) paPaePTsR0;
552	} aPTs[1];
553	} PGMMAPPING;
554	/** Pointer to structure for tracking GC Mappings. */
555	typedef struct PGMMAPPING *PPGMMAPPING;
556
557
558	/**
559	* Physical page access handler structure.
560	*
561	* This is used to keep track of physical address ranges
562	* which are being monitored in some kind of way.
563	*/
564	typedef struct PGMPHYSHANDLER
565	{
566	AVLROGCPHYSNODECORE Core;
567	/** Access type. */
568	PGMPHYSHANDLERTYPE enmType;
569	/** Number of pages to update. */
570	uint32_t cPages;
571	/** Set if we have pages that have been aliased. */
572	uint32_t cAliasedPages;
573	/** Set if we have pages that have temporarily been disabled. */
574	uint32_t cTmpOffPages;
575	/** Pointer to R3 callback function. */
576	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
577	/** User argument for R3 handlers. */
578	R3PTRTYPE(void *) pvUserR3;
579	/** Pointer to R0 callback function. */
580	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
581	/** User argument for R0 handlers. */
582	R0PTRTYPE(void *) pvUserR0;
583	/** Pointer to RC callback function. */
584	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
585	/** User argument for RC handlers. */
586	RCPTRTYPE(void *) pvUserRC;
587	/** Description / Name. For easing debugging. */
588	R3PTRTYPE(const char *) pszDesc;
589	#ifdef VBOX_WITH_STATISTICS
590	/** Profiling of this handler. */
591	STAMPROFILE Stat;
592	#endif
593	} PGMPHYSHANDLER;
594	/** Pointer to a physical page access handler structure. */
595	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
596
597
598	/**
599	* Cache node for the physical addresses covered by a virtual handler.
600	*/
601	typedef struct PGMPHYS2VIRTHANDLER
602	{
603	/** Core node for the tree based on physical ranges. */
604	AVLROGCPHYSNODECORE Core;
605	/** Offset from this struct to the PGMVIRTHANDLER structure. */
606	int32_t offVirtHandler;
607	/** Offset of the next alias relative to this one.
608	* Bit 0 is used for indicating whether we're in the tree.
609	* Bit 1 is used for indicating that we're the head node.
610	*/
611	int32_t offNextAlias;
612	} PGMPHYS2VIRTHANDLER;
613	/** Pointer to a phys to virtual handler structure. */
614	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
615
616	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
617	* node is in the tree. */
618	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
619	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
620	* node is in the head of an alias chain.
621	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
622	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
623	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
624	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
625
626
627	/**
628	* Virtual page access handler structure.
629	*
630	* This is used to keep track of virtual address ranges
631	* which are being monitored in some kind of way.
632	*/
633	typedef struct PGMVIRTHANDLER
634	{
635	/** Core node for the tree based on virtual ranges. */
636	AVLROGCPTRNODECORE Core;
637	/** Size of the range (in bytes). */
638	RTGCPTR cb;
639	/** Number of cache pages. */
640	uint32_t cPages;
641	/** Access type. */
642	PGMVIRTHANDLERTYPE enmType;
643	/** Pointer to the RC callback function. */
644	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
645	#if HC_ARCH_BITS == 64
646	RTRCPTR padding;
647	#endif
648	/** Pointer to the R3 callback function for invalidation. */
649	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
650	/** Pointer to the R3 callback function. */
651	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
652	/** Description / Name. For easing debugging. */
653	R3PTRTYPE(const char *) pszDesc;
654	#ifdef VBOX_WITH_STATISTICS
655	/** Profiling of this handler. */
656	STAMPROFILE Stat;
657	#endif
658	/** Array of cached physical addresses for the monitored ranged. */
659	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
660	} PGMVIRTHANDLER;
661	/** Pointer to a virtual page access handler structure. */
662	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
663
664
665	/** @name Page type predicates.
666	* @{ */
667	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
668	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
669	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
670	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
671	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
672	/** @} */
673
674
675	/**
676	* A Physical Guest Page tracking structure.
677	*
678	* The format of this structure is complicated because we have to fit a lot
679	* of information into as few bits as possible. The format is also subject
680	* to change (there is one coming up soon). Which means that for we'll be
681	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
682	* accesses to the structure.
683	*/
684	typedef union PGMPAGE
685	{
686	/** Structured view. */
687	struct
688	{
689	/** 1:0 - The physical handler state (PGM_PAGE_HNDL_PHYS_STATE_). /
690	uint64_t u2HandlerPhysStateY : 2;
691	/** 3:2 - Paging structure needed to map the page
692	* (PGM_PAGE_PDE_TYPE_). /
693	uint64_t u2PDETypeY : 2;
694	/** 4 - Indicator of dirty page for fault tolerance tracking. */
695	uint64_t fFTDirtyY : 1;
696	/** 5 - Flag indicating that a write monitored page was written to
697	* when set. */
698	uint64_t fWrittenToY : 1;
699	/** 7:6 - Unused. */
700	uint64_t u2Unused0 : 2;
701	/** 9:8 - The physical handler state (PGM_PAGE_HNDL_VIRT_STATE_). /
702	uint64_t u2HandlerVirtStateY : 2;
703	/** 11:10 - Unused. */
704	uint64_t u2Unused1 : 2;
705	/** 12:48 - The host physical frame number (shift left to get the
706	* address). */
707	uint64_t HCPhysFN : 36;
708	/** 50:48 - The page state. */
709	uint64_t uStateY : 3;
710	/** 51:53 - The page type (PGMPAGETYPE). */
711	uint64_t uTypeY : 3;
712	/** 63:54 - PTE index for usage tracking (page pool). */
713	uint64_t u10PteIdx : 10;
714
715	/** The GMM page ID. */
716	uint32_t idPage;
717	/** Usage tracking (page pool). */
718	uint16_t u16TrackingY;
719	/** The number of read locks on this page. */
720	uint8_t cReadLocksY;
721	/** The number of write locks on this page. */
722	uint8_t cWriteLocksY;
723	} s;
724
725	/** 64-bit integer view. */
726	uint64_t au64[2];
727	/** 16-bit view. */
728	uint32_t au32[4];
729	/** 16-bit view. */
730	uint16_t au16[8];
731	/** 8-bit view. */
732	uint8_t au8[16];
733	} PGMPAGE;
734	AssertCompileSize(PGMPAGE, 16);
735	/** Pointer to a physical guest page. */
736	typedef PGMPAGE *PPGMPAGE;
737	/** Pointer to a const physical guest page. */
738	typedef const PGMPAGE *PCPGMPAGE;
739	/** Pointer to a physical guest page pointer. */
740	typedef PPGMPAGE *PPPGMPAGE;
741
742
743	/**
744	* Clears the page structure.
745	* @param a_pPage Pointer to the physical guest page tracking structure.
746	*/
747	#define PGM_PAGE_CLEAR(a_pPage) \
748	do { \
749	(a_pPage)->au64[0] = 0; \
750	(a_pPage)->au64[1] = 0; \
751	} while (0)
752
753	/**
754	* Initializes the page structure.
755	* @param a_pPage Pointer to the physical guest page tracking structure.
756	*/
757	#define PGM_PAGE_INIT(a_pPage, a_HCPhys, a_idPage, a_uType, a_uState) \
758	do { \
759	RTHCPHYS SetHCPhysTmp = (a_HCPhys); \
760	AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
761	(a_pPage)->au64[0] = SetHCPhysTmp; \
762	(a_pPage)->au64[1] = 0; \
763	(a_pPage)->s.idPage = (a_idPage); \
764	(a_pPage)->s.uStateY = (a_uState); \
765	(a_pPage)->s.uTypeY = (a_uType); \
766	} while (0)
767
768	/**
769	* Initializes the page structure of a ZERO page.
770	* @param a_pPage Pointer to the physical guest page tracking structure.
771	* @param a_pVM The VM handle (for getting the zero page address).
772	* @param a_uType The page type (PGMPAGETYPE).
773	*/
774	#define PGM_PAGE_INIT_ZERO(a_pPage, a_pVM, a_uType) \
775	PGM_PAGE_INIT((a_pPage), (a_pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (a_uType), PGM_PAGE_STATE_ZERO)
776
777
778	/** @name The Page state, PGMPAGE::uStateY.
779	* @{ */
780	/** The zero page.
781	* This is a per-VM page that's never ever mapped writable. */
782	#define PGM_PAGE_STATE_ZERO 0
783	/** A allocated page.
784	* This is a per-VM page allocated from the page pool (or wherever
785	* we get MMIO2 pages from if the type is MMIO2).
786	*/
787	#define PGM_PAGE_STATE_ALLOCATED 1
788	/** A allocated page that's being monitored for writes.
789	* The shadow page table mappings are read-only. When a write occurs, the
790	* fWrittenTo member is set, the page remapped as read-write and the state
791	* moved back to allocated. */
792	#define PGM_PAGE_STATE_WRITE_MONITORED 2
793	/** The page is shared, aka. copy-on-write.
794	* This is a page that's shared with other VMs. */
795	#define PGM_PAGE_STATE_SHARED 3
796	/** The page is ballooned, so no longer available for this VM. */
797	#define PGM_PAGE_STATE_BALLOONED 4
798	/** @} */
799
800
801	/** Asserts lock ownership in some of the PGM_PAGE_XXX macros. */
802	#if defined(VBOX_STRICT) && 0 /** @todo triggers in pgmRZDynMapGCPageV2Inlined */
803	# define PGM_PAGE_ASSERT_LOCK(a_pVM) PGM_LOCK_ASSERT_OWNER(a_pVM)
804	#else
805	# define PGM_PAGE_ASSERT_LOCK(a_pVM) do { } while (0)
806	#endif
807
808	/**
809	* Gets the page state.
810	* @returns page state (PGM_PAGE_STATE_*).
811	* @param a_pPage Pointer to the physical guest page tracking structure.
812	*
813	* @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
814	* builds.
815	*/
816	#define PGM_PAGE_GET_STATE_NA(a_pPage) ( (a_pPage)->s.uStateY )
817	#if defined(__GNUC__) && defined(VBOX_STRICT)
818	# define PGM_PAGE_GET_STATE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_STATE_NA(a_pPage); })
819	#else
820	# define PGM_PAGE_GET_STATE PGM_PAGE_GET_STATE_NA
821	#endif
822
823	/**
824	* Sets the page state.
825	* @param a_pVM The VM handle, only used for lock ownership assertions.
826	* @param a_pPage Pointer to the physical guest page tracking structure.
827	* @param a_uState The new page state.
828	*/
829	#define PGM_PAGE_SET_STATE(a_pVM, a_pPage, a_uState) \
830	do { (a_pPage)->s.uStateY = (a_uState); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
831
832
833	/**
834	* Gets the host physical address of the guest page.
835	* @returns host physical address (RTHCPHYS).
836	* @param a_pPage Pointer to the physical guest page tracking structure.
837	*
838	* @remarks In strict builds on gcc platforms, this macro will make some ugly
839	* assumption about a valid pVM variable/parameter being in the
840	* current context. It will use this pVM variable to assert that the
841	* PGM lock is held. Use the PGM_PAGE_GET_HCPHYS_NA in contexts where
842	* pVM is not around.
843	*/
844	#if 0
845	# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->s.HCPhysFN << 12 )
846	# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA
847	#else
848	# define PGM_PAGE_GET_HCPHYS_NA(a_pPage) ( (a_pPage)->au64[0] & UINT64_C(0x0000fffffffff000) )
849	# if defined(__GNUC__) && defined(VBOX_STRICT)
850	# define PGM_PAGE_GET_HCPHYS(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_HCPHYS_NA(a_pPage); })
851	# else
852	# define PGM_PAGE_GET_HCPHYS PGM_PAGE_GET_HCPHYS_NA
853	# endif
854	#endif
855
856	/**
857	* Sets the host physical address of the guest page.
858	*
859	* @param a_pVM The VM handle, only used for lock ownership assertions.
860	* @param a_pPage Pointer to the physical guest page tracking structure.
861	* @param a_HCPhys The new host physical address.
862	*/
863	#define PGM_PAGE_SET_HCPHYS(a_pVM, a_pPage, a_HCPhys) \
864	do { \
865	RTHCPHYS const SetHCPhysTmp = (a_HCPhys); \
866	AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
867	(a_pPage)->s.HCPhysFN = SetHCPhysTmp >> 12; \
868	PGM_PAGE_ASSERT_LOCK(a_pVM); \
869	} while (0)
870
871	/**
872	* Get the Page ID.
873	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
874	* @param a_pPage Pointer to the physical guest page tracking structure.
875	*/
876	#define PGM_PAGE_GET_PAGEID(a_pPage) ( (uint32_t)(a_pPage)->s.idPage )
877
878	/**
879	* Sets the Page ID.
880	* @param a_pVM The VM handle, only used for lock ownership assertions.
881	* @param a_pPage Pointer to the physical guest page tracking structure.
882	* @param a_idPage The new page ID.
883	*/
884	#define PGM_PAGE_SET_PAGEID(a_pVM, a_pPage, a_idPage) \
885	do { \
886	(a_pPage)->s.idPage = (a_idPage); \
887	PGM_PAGE_ASSERT_LOCK(a_pVM); \
888	} while (0)
889
890	/**
891	* Get the Chunk ID.
892	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
893	* @param a_pPage Pointer to the physical guest page tracking structure.
894	*/
895	#define PGM_PAGE_GET_CHUNKID(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) >> GMM_CHUNKID_SHIFT )
896
897	/**
898	* Get the index of the page within the allocation chunk.
899	* @returns The page index.
900	* @param a_pPage Pointer to the physical guest page tracking structure.
901	*/
902	#define PGM_PAGE_GET_PAGE_IN_CHUNK(a_pPage) ( PGM_PAGE_GET_PAGEID(a_pPage) & GMM_PAGEID_IDX_MASK )
903
904	/**
905	* Gets the page type.
906	* @returns The page type.
907	* @param a_pPage Pointer to the physical guest page tracking structure.
908	*
909	* @remarks See PGM_PAGE_GET_HCPHYS_NA for remarks about GCC and strict
910	* builds.
911	*/
912	#define PGM_PAGE_GET_TYPE_NA(a_pPage) ( (a_pPage)->s.uTypeY )
913	#if defined(__GNUC__) && defined(VBOX_STRICT)
914	# define PGM_PAGE_GET_TYPE(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TYPE_NA(a_pPage); })
915	#else
916	# define PGM_PAGE_GET_TYPE PGM_PAGE_GET_TYPE_NA
917	#endif
918
919	/**
920	* Sets the page type.
921	*
922	* @param a_pVM The VM handle, only used for lock ownership assertions.
923	* @param a_pPage Pointer to the physical guest page tracking structure.
924	* @param a_enmType The new page type (PGMPAGETYPE).
925	*/
926	#define PGM_PAGE_SET_TYPE(a_pVM, a_pPage, a_enmType) \
927	do { (a_pPage)->s.uTypeY = (a_enmType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
928
929	/**
930	* Gets the page table index
931	* @returns The page table index.
932	* @param a_pPage Pointer to the physical guest page tracking structure.
933	*/
934	#define PGM_PAGE_GET_PTE_INDEX(a_pPage) ( (a_pPage)->s.u10PteIdx )
935
936	/**
937	* Sets the page table index.
938	* @param a_pVM The VM handle, only used for lock ownership assertions.
939	* @param a_pPage Pointer to the physical guest page tracking structure.
940	* @param a_iPte New page table index.
941	*/
942	#define PGM_PAGE_SET_PTE_INDEX(a_pVM, a_pPage, a_iPte) \
943	do { (a_pPage)->s.u10PteIdx = (a_iPte); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
944
945	/**
946	* Checks if the page is marked for MMIO.
947	* @returns true/false.
948	* @param a_pPage Pointer to the physical guest page tracking structure.
949	*/
950	#define PGM_PAGE_IS_MMIO(a_pPage) ( (a_pPage)->s.uTypeY == PGMPAGETYPE_MMIO )
951
952	/**
953	* Checks if the page is backed by the ZERO page.
954	* @returns true/false.
955	* @param a_pPage Pointer to the physical guest page tracking structure.
956	*/
957	#define PGM_PAGE_IS_ZERO(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ZERO )
958
959	/**
960	* Checks if the page is backed by a SHARED page.
961	* @returns true/false.
962	* @param a_pPage Pointer to the physical guest page tracking structure.
963	*/
964	#define PGM_PAGE_IS_SHARED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_SHARED )
965
966	/**
967	* Checks if the page is ballooned.
968	* @returns true/false.
969	* @param a_pPage Pointer to the physical guest page tracking structure.
970	*/
971	#define PGM_PAGE_IS_BALLOONED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_BALLOONED )
972
973	/**
974	* Checks if the page is allocated.
975	* @returns true/false.
976	* @param a_pPage Pointer to the physical guest page tracking structure.
977	*/
978	#define PGM_PAGE_IS_ALLOCATED(a_pPage) ( (a_pPage)->s.uStateY == PGM_PAGE_STATE_ALLOCATED )
979
980	/**
981	* Marks the page as written to (for GMM change monitoring).
982	* @param a_pVM The VM handle, only used for lock ownership assertions.
983	* @param a_pPage Pointer to the physical guest page tracking structure.
984	*/
985	#define PGM_PAGE_SET_WRITTEN_TO(a_pVM, a_pPage) \
986	do { (a_pPage)->au8[1] \|= UINT8_C(0x80); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0) /// FIXME FIXME
987
988	/**
989	* Clears the written-to indicator.
990	* @param a_pVM The VM handle, only used for lock ownership assertions.
991	* @param a_pPage Pointer to the physical guest page tracking structure.
992	*/
993	#define PGM_PAGE_CLEAR_WRITTEN_TO(a_pVM, a_pPage) \
994	do { (a_pPage)->s.fWrittenToY = 0; PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
995
996	/**
997	* Checks if the page was marked as written-to.
998	* @returns true/false.
999	* @param a_pPage Pointer to the physical guest page tracking structure.
1000	*/
1001	#define PGM_PAGE_IS_WRITTEN_TO(a_pPage) ( (a_pPage)->s.fWrittenToY )
1002
1003	/**
1004	* Marks the page as dirty for FTM
1005	* @param a_pPage Pointer to the physical guest page tracking structure.
1006	*/
1007	#define PGM_PAGE_SET_FT_DIRTY(a_pPage) do { (a_pPage)->s.fFTDirtyY = 1; } while (0)
1008
1009	/**
1010	* Clears the FTM dirty indicator
1011	* @param a_pPage Pointer to the physical guest page tracking structure.
1012	*/
1013	#define PGM_PAGE_CLEAR_FT_DIRTY(a_pPage) do { (a_pPage)->s.fFTDirtyY = 0; } while (0)
1014
1015	/**
1016	* Checks if the page was marked as dirty for FTM
1017	* @returns true/false.
1018	* @param a_pPage Pointer to the physical guest page tracking structure.
1019	*/
1020	#define PGM_PAGE_IS_FT_DIRTY(a_pPage) ( (a_pPage)->s.fFTDirtyY )
1021
1022
1023	/** @name PT usage values (PGMPAGE::u2PDEType).
1024	*
1025	* @{ */
1026	/** Either as a PT or PDE. */
1027	#define PGM_PAGE_PDE_TYPE_DONTCARE 0
1028	/** Must use a page table to map the range. */
1029	#define PGM_PAGE_PDE_TYPE_PT 1
1030	/** Can use a page directory entry to map the continuous range. */
1031	#define PGM_PAGE_PDE_TYPE_PDE 2
1032	/** Can use a page directory entry to map the continuous range - temporarily disabled (by page monitoring). */
1033	#define PGM_PAGE_PDE_TYPE_PDE_DISABLED 3
1034	/** @} */
1035
1036	/**
1037	* Set the PDE type of the page
1038	* @param a_pVM The VM handle, only used for lock ownership assertions.
1039	* @param a_pPage Pointer to the physical guest page tracking structure.
1040	* @param a_uType PGM_PAGE_PDE_TYPE_*.
1041	*/
1042	#define PGM_PAGE_SET_PDE_TYPE(a_pVM, a_pPage, a_uType) \
1043	do { (a_pPage)->s.u2PDETypeY = (a_uType); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
1044
1045	/**
1046	* Checks if the page was marked being part of a large page
1047	* @returns true/false.
1048	* @param a_pPage Pointer to the physical guest page tracking structure.
1049	*/
1050	#define PGM_PAGE_GET_PDE_TYPE(a_pPage) ( (a_pPage)->s.u2PDETypeY )
1051
1052	/** Enabled optimized access handler tests.
1053	* These optimizations makes ASSUMPTIONS about the state values and the s1
1054	* layout. When enabled, the compiler should normally generate more compact
1055	* code.
1056	*/
1057	#define PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS 1
1058
1059	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
1060	*
1061	* @remarks The values are assigned in order of priority, so we can calculate
1062	* the correct state for a page with different handlers installed.
1063	* @{ */
1064	/** No handler installed. */
1065	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
1066	/** Monitoring is temporarily disabled. */
1067	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
1068	/** Write access is monitored. */
1069	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
1070	/** All access is monitored. */
1071	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
1072	/** @} */
1073
1074	/**
1075	* Gets the physical access handler state of a page.
1076	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
1077	* @param a_pPage Pointer to the physical guest page tracking structure.
1078	*/
1079	#define PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) ( (a_pPage)->s.u2HandlerPhysStateY )
1080
1081	/**
1082	* Sets the physical access handler state of a page.
1083	* @param a_pPage Pointer to the physical guest page tracking structure.
1084	* @param a_uState The new state value.
1085	*/
1086	#define PGM_PAGE_SET_HNDL_PHYS_STATE(a_pPage, a_uState) \
1087	do { (a_pPage)->s.u2HandlerPhysStateY = (a_uState); } while (0)
1088
1089	/**
1090	* Checks if the page has any physical access handlers, including temporarily disabled ones.
1091	* @returns true/false
1092	* @param a_pPage Pointer to the physical guest page tracking structure.
1093	*/
1094	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(a_pPage) \
1095	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
1096
1097	/**
1098	* Checks if the page has any active physical access handlers.
1099	* @returns true/false
1100	* @param a_pPage Pointer to the physical guest page tracking structure.
1101	*/
1102	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(a_pPage) \
1103	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
1104
1105
1106	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateY).
1107	*
1108	* @remarks The values are assigned in order of priority, so we can calculate
1109	* the correct state for a page with different handlers installed.
1110	* @{ */
1111	/** No handler installed. */
1112	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
1113	/* 1 is reserved so the lineup is identical with the physical ones. */
1114	/** Write access is monitored. */
1115	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
1116	/** All access is monitored. */
1117	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
1118	/** @} */
1119
1120	/**
1121	* Gets the virtual access handler state of a page.
1122	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
1123	* @param a_pPage Pointer to the physical guest page tracking structure.
1124	*/
1125	#define PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) ( (a_pPage)->s.u2HandlerVirtStateY )
1126
1127	/**
1128	* Sets the virtual access handler state of a page.
1129	* @param a_pPage Pointer to the physical guest page tracking structure.
1130	* @param a_uState The new state value.
1131	*/
1132	#define PGM_PAGE_SET_HNDL_VIRT_STATE(a_pPage, a_uState) \
1133	do { (a_pPage)->s.u2HandlerVirtStateY = (a_uState); } while (0)
1134
1135	/**
1136	* Checks if the page has any virtual access handlers.
1137	* @returns true/false
1138	* @param a_pPage Pointer to the physical guest page tracking structure.
1139	*/
1140	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(a_pPage) \
1141	( PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
1142
1143	/**
1144	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
1145	* virtual handlers.
1146	* @returns true/false
1147	* @param a_pPage Pointer to the physical guest page tracking structure.
1148	*/
1149	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(a_pPage) \
1150	PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(a_pPage)
1151
1152
1153	/**
1154	* Checks if the page has any access handlers, including temporarily disabled ones.
1155	* @returns true/false
1156	* @param a_pPage Pointer to the physical guest page tracking structure.
1157	*/
1158	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1159	# define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \
1160	( ((a_pPage)->au32[0] & UINT16_C(0x0303)) != 0 )
1161	#else
1162	# define PGM_PAGE_HAS_ANY_HANDLERS(a_pPage) \
1163	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE \
1164	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
1165	#endif
1166
1167	/**
1168	* Checks if the page has any active access handlers.
1169	* @returns true/false
1170	* @param a_pPage Pointer to the physical guest page tracking structure.
1171	*/
1172	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1173	# define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \
1174	( ((a_pPage)->au32[0] & UINT16_C(0x0202)) != 0 )
1175	#else
1176	# define PGM_PAGE_HAS_ACTIVE_HANDLERS(a_pPage) \
1177	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
1178	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
1179	#endif
1180
1181	/**
1182	* Checks if the page has any active access handlers catching all accesses.
1183	* @returns true/false
1184	* @param a_pPage Pointer to the physical guest page tracking structure.
1185	*/
1186	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1187	# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \
1188	( ( ((a_pPage)->au8[0] \| (a_pPage)->au8[1]) & UINT8_C(0x3) ) \
1189	== PGM_PAGE_HNDL_PHYS_STATE_ALL )
1190	#else
1191	# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(a_pPage) \
1192	( PGM_PAGE_GET_HNDL_PHYS_STATE(a_pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL \
1193	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(a_pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL )
1194	#endif
1195
1196
1197	/** @def PGM_PAGE_GET_TRACKING
1198	* Gets the packed shadow page pool tracking data associated with a guest page.
1199	* @returns uint16_t containing the data.
1200	* @param a_pPage Pointer to the physical guest page tracking structure.
1201	*/
1202	#define PGM_PAGE_GET_TRACKING_NA(a_pPage) ( (a_pPage)->s.u16TrackingY )
1203	#if defined(__GNUC__) && defined(VBOX_STRICT)
1204	# define PGM_PAGE_GET_TRACKING(a_pPage) __extension__ ({ PGM_PAGE_ASSERT_LOCK(pVM); PGM_PAGE_GET_TRACKING_NA(a_pPage); })
1205	#else
1206	# define PGM_PAGE_GET_TRACKING PGM_PAGE_GET_TRACKING_NA
1207	#endif
1208
1209	/** @def PGM_PAGE_SET_TRACKING
1210	* Sets the packed shadow page pool tracking data associated with a guest page.
1211	* @param a_pVM The VM handle, only used for lock ownership assertions.
1212	* @param a_pPage Pointer to the physical guest page tracking structure.
1213	* @param a_u16TrackingData The tracking data to store.
1214	*/
1215	#define PGM_PAGE_SET_TRACKING(a_pVM, a_pPage, a_u16TrackingData) \
1216	do { (a_pPage)->s.u16TrackingY = (a_u16TrackingData); PGM_PAGE_ASSERT_LOCK(a_pVM); } while (0)
1217
1218	/** @def PGM_PAGE_GET_TD_CREFS
1219	* Gets the @a cRefs tracking data member.
1220	* @returns cRefs.
1221	* @param a_pPage Pointer to the physical guest page tracking structure.
1222	*/
1223	#define PGM_PAGE_GET_TD_CREFS(a_pPage) \
1224	((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1225	#define PGM_PAGE_GET_TD_CREFS_NA(a_pPage) \
1226	((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1227
1228	/** @def PGM_PAGE_GET_TD_IDX
1229	* Gets the @a idx tracking data member.
1230	* @returns idx.
1231	* @param a_pPage Pointer to the physical guest page tracking structure.
1232	*/
1233	#define PGM_PAGE_GET_TD_IDX(a_pPage) \
1234	((PGM_PAGE_GET_TRACKING(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1235	#define PGM_PAGE_GET_TD_IDX_NA(a_pPage) \
1236	((PGM_PAGE_GET_TRACKING_NA(a_pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1237
1238
1239	/** Max number of locks on a page. */
1240	#define PGM_PAGE_MAX_LOCKS UINT8_C(254)
1241
1242	/** Get the read lock count.
1243	* @returns count.
1244	* @param a_pPage Pointer to the physical guest page tracking structure.
1245	*/
1246	#define PGM_PAGE_GET_READ_LOCKS(a_pPage) ( (a_pPage)->s.cReadLocksY )
1247
1248	/** Get the write lock count.
1249	* @returns count.
1250	* @param a_pPage Pointer to the physical guest page tracking structure.
1251	*/
1252	#define PGM_PAGE_GET_WRITE_LOCKS(a_pPage) ( (a_pPage)->s.cWriteLocksY )
1253
1254	/** Decrement the read lock counter.
1255	* @param a_pPage Pointer to the physical guest page tracking structure.
1256	*/
1257	#define PGM_PAGE_DEC_READ_LOCKS(a_pPage) do { --(a_pPage)->s.cReadLocksY; } while (0)
1258
1259	/** Decrement the write lock counter.
1260	* @param a_pPage Pointer to the physical guest page tracking structure.
1261	*/
1262	#define PGM_PAGE_DEC_WRITE_LOCKS(a_pPage) do { --(a_pPage)->s.cWriteLocksY; } while (0)
1263
1264	/** Increment the read lock counter.
1265	* @param a_pPage Pointer to the physical guest page tracking structure.
1266	*/
1267	#define PGM_PAGE_INC_READ_LOCKS(a_pPage) do { ++(a_pPage)->s.cReadLocksY; } while (0)
1268
1269	/** Increment the write lock counter.
1270	* @param a_pPage Pointer to the physical guest page tracking structure.
1271	*/
1272	#define PGM_PAGE_INC_WRITE_LOCKS(a_pPage) do { ++(a_pPage)->s.cWriteLocksY; } while (0)
1273
1274
1275	#if 0
1276	/** Enables sanity checking of write monitoring using CRC-32. */
1277	# define PGMLIVESAVERAMPAGE_WITH_CRC32
1278	#endif
1279
1280	/**
1281	* Per page live save tracking data.
1282	*/
1283	typedef struct PGMLIVESAVERAMPAGE
1284	{
1285	/** Number of times it has been dirtied. */
1286	uint32_t cDirtied : 24;
1287	/** Whether it is currently dirty. */
1288	uint32_t fDirty : 1;
1289	/** Ignore the page.
1290	* This is used for pages that has been MMIO, MMIO2 or ROM pages once. We will
1291	* deal with these after pausing the VM and DevPCI have said it bit about
1292	* remappings. */
1293	uint32_t fIgnore : 1;
1294	/** Was a ZERO page last time around. */
1295	uint32_t fZero : 1;
1296	/** Was a SHARED page last time around. */
1297	uint32_t fShared : 1;
1298	/** Whether the page is/was write monitored in a previous pass. */
1299	uint32_t fWriteMonitored : 1;
1300	/** Whether the page is/was write monitored earlier in this pass. */
1301	uint32_t fWriteMonitoredJustNow : 1;
1302	/** Bits reserved for future use. */
1303	uint32_t u2Reserved : 2;
1304	#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1305	/** CRC-32 for the page. This is for internal consistency checks. */
1306	uint32_t u32Crc;
1307	#endif
1308	} PGMLIVESAVERAMPAGE;
1309	#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1310	AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
1311	#else
1312	AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
1313	#endif
1314	/** Pointer to the per page live save tracking data. */
1315	typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
1316
1317	/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
1318	#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
1319
1320
1321	/**
1322	* RAM range for GC Phys to HC Phys conversion.
1323	*
1324	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
1325	* conversions too, but we'll let MM handle that for now.
1326	*
1327	* This structure is used by linked lists in both GC and HC.
1328	*/
1329	typedef struct PGMRAMRANGE
1330	{
1331	/** Start of the range. Page aligned. */
1332	RTGCPHYS GCPhys;
1333	/** Size of the range. (Page aligned of course). */
1334	RTGCPHYS cb;
1335	/** Pointer to the next RAM range - for R3. */
1336	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
1337	/** Pointer to the next RAM range - for R0. */
1338	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
1339	/** Pointer to the next RAM range - for RC. */
1340	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
1341	/** PGM_RAM_RANGE_FLAGS_* flags. */
1342	uint32_t fFlags;
1343	/** Last address in the range (inclusive). Page aligned (-1). */
1344	RTGCPHYS GCPhysLast;
1345	/** Start of the HC mapping of the range. This is only used for MMIO2. */
1346	R3PTRTYPE(void *) pvR3;
1347	/** Live save per page tracking data. */
1348	R3PTRTYPE(PPGMLIVESAVERAMPAGE) paLSPages;
1349	/** The range description. */
1350	R3PTRTYPE(const char *) pszDesc;
1351	/** Pointer to self - R0 pointer. */
1352	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1353	/** Pointer to self - RC pointer. */
1354	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1355
1356	/** Alignment padding. */
1357	RTRCPTR Alignment0;
1358	/** Pointer to the left search three node - ring-3 context. */
1359	R3PTRTYPE(struct PGMRAMRANGE *) pLeftR3;
1360	/** Pointer to the right search three node - ring-3 context. */
1361	R3PTRTYPE(struct PGMRAMRANGE *) pRightR3;
1362	/** Pointer to the left search three node - ring-0 context. */
1363	R0PTRTYPE(struct PGMRAMRANGE *) pLeftR0;
1364	/** Pointer to the right search three node - ring-0 context. */
1365	R0PTRTYPE(struct PGMRAMRANGE *) pRightR0;
1366	/** Pointer to the left search three node - raw-mode context. */
1367	RCPTRTYPE(struct PGMRAMRANGE *) pLeftRC;
1368	/** Pointer to the right search three node - raw-mode context. */
1369	RCPTRTYPE(struct PGMRAMRANGE *) pRightRC;
1370
1371	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1372	#if HC_ARCH_BITS == 32
1373	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 0];
1374	#endif
1375	/** Array of physical guest page tracking structures. */
1376	PGMPAGE aPages[1];
1377	} PGMRAMRANGE;
1378	/** Pointer to RAM range for GC Phys to HC Phys conversion. */
1379	typedef PGMRAMRANGE *PPGMRAMRANGE;
1380
1381	/** @name PGMRAMRANGE::fFlags
1382	* @{ */
1383	/** The RAM range is floating around as an independent guest mapping. */
1384	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1385	/** Ad hoc RAM range for an ROM mapping. */
1386	#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM RT_BIT(21)
1387	/** Ad hoc RAM range for an MMIO mapping. */
1388	#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO RT_BIT(22)
1389	/** Ad hoc RAM range for an MMIO2 mapping. */
1390	#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2 RT_BIT(23)
1391	/** @} */
1392
1393	/** Tests if a RAM range is an ad hoc one or not.
1394	* @returns true/false.
1395	* @param pRam The RAM range.
1396	*/
1397	#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
1398	(!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM \| PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO \| PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2) ) )
1399
1400	/** The number of entries in the RAM range TLBs (there is one for each
1401	* context). Must be a power of two. */
1402	#define PGM_RAMRANGE_TLB_ENTRIES 8
1403
1404	/**
1405	* Calculates the RAM range TLB index for the physical address.
1406	*
1407	* @returns RAM range TLB index.
1408	* @param GCPhys The guest physical address.
1409	*/
1410	#define PGM_RAMRANGE_TLB_IDX(a_GCPhys) ( ((a_GCPhys) >> 20) & (PGM_RAMRANGE_TLB_ENTRIES - 1) )
1411
1412
1413
1414	/**
1415	* Per page tracking structure for ROM image.
1416	*
1417	* A ROM image may have a shadow page, in which case we may have two pages
1418	* backing it. This structure contains the PGMPAGE for both while
1419	* PGMRAMRANGE have a copy of the active one. It is important that these
1420	* aren't out of sync in any regard other than page pool tracking data.
1421	*/
1422	typedef struct PGMROMPAGE
1423	{
1424	/** The page structure for the virgin ROM page. */
1425	PGMPAGE Virgin;
1426	/** The page structure for the shadow RAM page. */
1427	PGMPAGE Shadow;
1428	/** The current protection setting. */
1429	PGMROMPROT enmProt;
1430	/** Live save status information. Makes use of unused alignment space. */
1431	struct
1432	{
1433	/** The previous protection value. */
1434	uint8_t u8Prot;
1435	/** Written to flag set by the handler. */
1436	bool fWrittenTo;
1437	/** Whether the shadow page is dirty or not. */
1438	bool fDirty;
1439	/** Whether it was dirtied in the recently. */
1440	bool fDirtiedRecently;
1441	} LiveSave;
1442	} PGMROMPAGE;
1443	AssertCompileSizeAlignment(PGMROMPAGE, 8);
1444	/** Pointer to a ROM page tracking structure. */
1445	typedef PGMROMPAGE *PPGMROMPAGE;
1446
1447
1448	/**
1449	* A registered ROM image.
1450	*
1451	* This is needed to keep track of ROM image since they generally intrude
1452	* into a PGMRAMRANGE. It also keeps track of additional info like the
1453	* two page sets (read-only virgin and read-write shadow), the current
1454	* state of each page.
1455	*
1456	* Because access handlers cannot easily be executed in a different
1457	* context, the ROM ranges needs to be accessible and in all contexts.
1458	*/
1459	typedef struct PGMROMRANGE
1460	{
1461	/** Pointer to the next range - R3. */
1462	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1463	/** Pointer to the next range - R0. */
1464	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1465	/** Pointer to the next range - RC. */
1466	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1467	/** Pointer alignment */
1468	RTRCPTR RCPtrAlignment;
1469	/** Address of the range. */
1470	RTGCPHYS GCPhys;
1471	/** Address of the last byte in the range. */
1472	RTGCPHYS GCPhysLast;
1473	/** Size of the range. */
1474	RTGCPHYS cb;
1475	/** The flags (PGMPHYS_ROM_FLAGS_). /
1476	uint32_t fFlags;
1477	/** The saved state range ID. */
1478	uint8_t idSavedState;
1479	/** Alignment padding. */
1480	uint8_t au8Alignment[3];
1481	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1482	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 5 : 1];
1483	/** The size bits pvOriginal points to. */
1484	uint32_t cbOriginal;
1485	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1486	* This is used for strictness checks. */
1487	R3PTRTYPE(const void *) pvOriginal;
1488	/** The ROM description. */
1489	R3PTRTYPE(const char *) pszDesc;
1490	/** The per page tracking structures. */
1491	PGMROMPAGE aPages[1];
1492	} PGMROMRANGE;
1493	/** Pointer to a ROM range. */
1494	typedef PGMROMRANGE *PPGMROMRANGE;
1495
1496
1497	/**
1498	* Live save per page data for an MMIO2 page.
1499	*
1500	* Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
1501	* of MMIO2 pages. The current approach is using some optimistic SHA-1 +
1502	* CRC-32 for detecting changes as well as special handling of zero pages. This
1503	* is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
1504	* for speeding things up. (We're using SHA-1 and not SHA-256 or SHA-512
1505	* because of speed (2.5x and 6x slower).)
1506	*
1507	* @todo Implement dirty MMIO2 page reporting that can be enabled during live
1508	* save but normally is disabled. Since we can write monitor guest
1509	* accesses on our own, we only need this for host accesses. Shouldn't be
1510	* too difficult for DevVGA, VMMDev might be doable, the planned
1511	* networking fun will be fun since it involves ring-0.
1512	*/
1513	typedef struct PGMLIVESAVEMMIO2PAGE
1514	{
1515	/** Set if the page is considered dirty. */
1516	bool fDirty;
1517	/** The number of scans this page has remained unchanged for.
1518	* Only updated for dirty pages. */
1519	uint8_t cUnchangedScans;
1520	/** Whether this page was zero at the last scan. */
1521	bool fZero;
1522	/** Alignment padding. */
1523	bool fReserved;
1524	/** CRC-32 for the first half of the page.
1525	* This is used together with u32CrcH2 to quickly detect changes in the page
1526	* during the non-final passes. */
1527	uint32_t u32CrcH1;
1528	/** CRC-32 for the second half of the page. */
1529	uint32_t u32CrcH2;
1530	/** SHA-1 for the saved page.
1531	* This is used in the final pass to skip pages without changes. */
1532	uint8_t abSha1Saved[RTSHA1_HASH_SIZE];
1533	} PGMLIVESAVEMMIO2PAGE;
1534	/** Pointer to a live save status data for an MMIO2 page. */
1535	typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
1536
1537	/**
1538	* A registered MMIO2 (= Device RAM) range.
1539	*
1540	* There are a few reason why we need to keep track of these
1541	* registrations. One of them is the deregistration & cleanup stuff,
1542	* while another is that the PGMRAMRANGE associated with such a region may
1543	* have to be removed from the ram range list.
1544	*
1545	* Overlapping with a RAM range has to be 100% or none at all. The pages
1546	* in the existing RAM range must not be ROM nor MMIO. A guru meditation
1547	* will be raised if a partial overlap or an overlap of ROM pages is
1548	* encountered. On an overlap we will free all the existing RAM pages and
1549	* put in the ram range pages instead.
1550	*/
1551	typedef struct PGMMMIO2RANGE
1552	{
1553	/** The owner of the range. (a device) */
1554	PPDMDEVINSR3 pDevInsR3;
1555	/** Pointer to the ring-3 mapping of the allocation. */
1556	RTR3PTR pvR3;
1557	/** Pointer to the next range - R3. */
1558	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1559	/** Whether it's mapped or not. */
1560	bool fMapped;
1561	/** Whether it's overlapping or not. */
1562	bool fOverlapping;
1563	/** The PCI region number.
1564	* @remarks This ASSUMES that nobody will ever really need to have multiple
1565	* PCI devices with matching MMIO region numbers on a single device. */
1566	uint8_t iRegion;
1567	/** The saved state range ID. */
1568	uint8_t idSavedState;
1569	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundary. */
1570	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 12 : 12];
1571	/** Live save per page tracking data. */
1572	R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE) paLSPages;
1573	/** The associated RAM range. */
1574	PGMRAMRANGE RamRange;
1575	} PGMMMIO2RANGE;
1576	/** Pointer to a MMIO2 range. */
1577	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1578
1579
1580
1581
1582	/**
1583	* PGMPhysRead/Write cache entry
1584	*/
1585	typedef struct PGMPHYSCACHEENTRY
1586	{
1587	/** R3 pointer to physical page. */
1588	R3PTRTYPE(uint8_t *) pbR3;
1589	/** GC Physical address for cache entry */
1590	RTGCPHYS GCPhys;
1591	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1592	RTGCPHYS u32Padding0; /*< alignment padding. /
1593	#endif
1594	} PGMPHYSCACHEENTRY;
1595
1596	/**
1597	* PGMPhysRead/Write cache to reduce REM memory access overhead
1598	*/
1599	typedef struct PGMPHYSCACHE
1600	{
1601	/** Bitmap of valid cache entries */
1602	uint64_t aEntries;
1603	/** Cache entries */
1604	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1605	} PGMPHYSCACHE;
1606
1607
1608	/** Pointer to an allocation chunk ring-3 mapping. */
1609	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1610	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1611	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1612
1613	/**
1614	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1615	*
1616	* The primary tree (Core) uses the chunk id as key.
1617	*/
1618	typedef struct PGMCHUNKR3MAP
1619	{
1620	/** The key is the chunk id. */
1621	AVLU32NODECORE Core;
1622	/** The current age thingy. */
1623	uint32_t iAge;
1624	/** The current reference count. */
1625	uint32_t volatile cRefs;
1626	/** The current permanent reference count. */
1627	uint32_t volatile cPermRefs;
1628	/** The mapping address. */
1629	void *pv;
1630	} PGMCHUNKR3MAP;
1631
1632	/**
1633	* Allocation chunk ring-3 mapping TLB entry.
1634	*/
1635	typedef struct PGMCHUNKR3MAPTLBE
1636	{
1637	/** The chunk id. */
1638	uint32_t volatile idChunk;
1639	#if HC_ARCH_BITS == 64
1640	uint32_t u32Padding; /*< alignment padding. /
1641	#endif
1642	/** The chunk map. */
1643	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1644	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1645	#else
1646	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1647	#endif
1648	} PGMCHUNKR3MAPTLBE;
1649	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1650	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1651
1652	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1653	* @remark Must be a power of two value. */
1654	#define PGM_CHUNKR3MAPTLB_ENTRIES 64
1655
1656	/**
1657	* Allocation chunk ring-3 mapping TLB.
1658	*
1659	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1660	* At first glance this might look kinda odd since AVL trees are
1661	* supposed to give the most optimal lookup times of all trees
1662	* due to their balancing. However, take a tree with 1023 nodes
1663	* in it, that's 10 levels, meaning that most searches has to go
1664	* down 9 levels before they find what they want. This isn't fast
1665	* compared to a TLB hit. There is the factor of cache misses,
1666	* and of course the problem with trees and branch prediction.
1667	* This is why we use TLBs in front of most of the trees.
1668	*
1669	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1670	* difficult when we switch to the new inlined AVL trees (from kStuff).
1671	*/
1672	typedef struct PGMCHUNKR3MAPTLB
1673	{
1674	/** The TLB entries. */
1675	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1676	} PGMCHUNKR3MAPTLB;
1677
1678	/**
1679	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1680	* @returns Chunk TLB index.
1681	* @param idChunk The Chunk ID.
1682	*/
1683	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1684
1685
1686	/**
1687	* Ring-3 guest page mapping TLB entry.
1688	* @remarks used in ring-0 as well at the moment.
1689	*/
1690	typedef struct PGMPAGER3MAPTLBE
1691	{
1692	/** Address of the page. */
1693	RTGCPHYS volatile GCPhys;
1694	/** The guest page. */
1695	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1696	R3PTRTYPE(PPGMPAGE) volatile pPage;
1697	#else
1698	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1699	#endif
1700	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1701	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1702	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1703	#else
1704	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1705	#endif
1706	/** The address */
1707	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1708	R3PTRTYPE(void *) volatile pv;
1709	#else
1710	R3R0PTRTYPE(void *) volatile pv;
1711	#endif
1712	#if HC_ARCH_BITS == 32
1713	uint32_t u32Padding; /*< alignment padding. /
1714	#endif
1715	} PGMPAGER3MAPTLBE;
1716	/** Pointer to an entry in the HC physical TLB. */
1717	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1718
1719
1720	/** The number of entries in the ring-3 guest page mapping TLB.
1721	* @remarks The value must be a power of two. */
1722	#define PGM_PAGER3MAPTLB_ENTRIES 256
1723
1724	/**
1725	* Ring-3 guest page mapping TLB.
1726	* @remarks used in ring-0 as well at the moment.
1727	*/
1728	typedef struct PGMPAGER3MAPTLB
1729	{
1730	/** The TLB entries. */
1731	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1732	} PGMPAGER3MAPTLB;
1733	/** Pointer to the ring-3 guest page mapping TLB. */
1734	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1735
1736	/**
1737	* Calculates the index of the TLB entry for the specified guest page.
1738	* @returns Physical TLB index.
1739	* @param GCPhys The guest physical address.
1740	*/
1741	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1742
1743
1744	/**
1745	* Raw-mode context dynamic mapping cache entry.
1746	*
1747	* Because of raw-mode context being reloctable and all relocations are applied
1748	* in ring-3, this has to be defined here and be RC specific.
1749	*
1750	* @sa PGMRZDYNMAPENTRY, PGMR0DYNMAPENTRY.
1751	*/
1752	typedef struct PGMRCDYNMAPENTRY
1753	{
1754	/** The physical address of the currently mapped page.
1755	* This is duplicate for three reasons: cache locality, cache policy of the PT
1756	* mappings and sanity checks. */
1757	RTHCPHYS HCPhys;
1758	/** Pointer to the page. */
1759	RTRCPTR pvPage;
1760	/** The number of references. */
1761	int32_t volatile cRefs;
1762	/** PTE pointer union. */
1763	struct PGMRCDYNMAPENTRY_PPTE
1764	{
1765	/** PTE pointer, 32-bit legacy version. */
1766	RCPTRTYPE(PX86PTE) pLegacy;
1767	/** PTE pointer, PAE version. */
1768	RCPTRTYPE(PX86PTEPAE) pPae;
1769	} uPte;
1770	} PGMRCDYNMAPENTRY;
1771	/** Pointer to a dynamic mapping cache entry for the raw-mode context. */
1772	typedef PGMRCDYNMAPENTRY *PPGMRCDYNMAPENTRY;
1773
1774
1775	/**
1776	* Dynamic mapping cache for the raw-mode context.
1777	*
1778	* This is initialized during VMMRC init based upon the pbDynPageMapBaseGC and
1779	* paDynPageMap* PGM members. However, it has to be defined in PGMInternal.h
1780	* so that we can perform relocations from PGMR3Relocate. This has the
1781	* consequence that we must have separate ring-0 and raw-mode context versions
1782	* of this struct even if they share the basic elements.
1783	*
1784	* @sa PPGMRZDYNMAP, PGMR0DYNMAP.
1785	*/
1786	typedef struct PGMRCDYNMAP
1787	{
1788	/** The usual magic number / eye catcher (PGMRZDYNMAP_MAGIC). */
1789	uint32_t u32Magic;
1790	/** Array for tracking and managing the pages. */
1791	RCPTRTYPE(PPGMRCDYNMAPENTRY) paPages;
1792	/** The cache size given as a number of pages. */
1793	uint32_t cPages;
1794	/** The current load.
1795	* This does not include guard pages. */
1796	uint32_t cLoad;
1797	/** The max load ever.
1798	* This is maintained to get trigger adding of more mapping space. */
1799	uint32_t cMaxLoad;
1800	/** The number of guard pages. */
1801	uint32_t cGuardPages;
1802	/** The number of users (protected by hInitLock). */
1803	uint32_t cUsers;
1804	} PGMRCDYNMAP;
1805	/** Pointer to the dynamic cache for the raw-mode context. */
1806	typedef PGMRCDYNMAP *PPGMRCDYNMAP;
1807
1808
1809	/**
1810	* Mapping cache usage set entry.
1811	*
1812	* @remarks 16-bit ints was chosen as the set is not expected to be used beyond
1813	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1814	* cache. If it's extended to include ring-3, well, then something
1815	* will have be changed here...
1816	*/
1817	typedef struct PGMMAPSETENTRY
1818	{
1819	/** Pointer to the page. */
1820	#ifndef IN_RC
1821	RTR0PTR pvPage;
1822	#else
1823	RTRCPTR pvPage;
1824	# if HC_ARCH_BITS == 64
1825	uint32_t u32Alignment2;
1826	# endif
1827	#endif
1828	/** The mapping cache index. */
1829	uint16_t iPage;
1830	/** The number of references.
1831	* The max is UINT16_MAX - 1. */
1832	uint16_t cRefs;
1833	/** The number inlined references.
1834	* The max is UINT16_MAX - 1. */
1835	uint16_t cInlinedRefs;
1836	/** Unreferences. */
1837	uint16_t cUnrefs;
1838
1839	#if HC_ARCH_BITS == 32
1840	uint32_t u32Alignment1;
1841	#endif
1842	/** The physical address for this entry. */
1843	RTHCPHYS HCPhys;
1844	} PGMMAPSETENTRY;
1845	AssertCompileMemberOffset(PGMMAPSETENTRY, iPage, RT_MAX(sizeof(RTR0PTR), sizeof(RTRCPTR)));
1846	AssertCompileMemberAlignment(PGMMAPSETENTRY, HCPhys, sizeof(RTHCPHYS));
1847	/** Pointer to a mapping cache usage set entry. */
1848	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1849
1850	/**
1851	* Mapping cache usage set.
1852	*
1853	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1854	* done during exits / traps. The set is
1855	*/
1856	typedef struct PGMMAPSET
1857	{
1858	/** The number of occupied entries.
1859	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1860	* dynamic mappings. */
1861	uint32_t cEntries;
1862	/** The start of the current subset.
1863	* This is UINT32_MAX if no subset is currently open. */
1864	uint32_t iSubset;
1865	/** The index of the current CPU, only valid if the set is open. */
1866	int32_t iCpu;
1867	uint32_t alignment;
1868	/** The entries. */
1869	PGMMAPSETENTRY aEntries[64];
1870	/** HCPhys -> iEntry fast lookup table.
1871	* Use PGMMAPSET_HASH for hashing.
1872	* The entries may or may not be valid, check against cEntries. */
1873	uint8_t aiHashTable[128];
1874	} PGMMAPSET;
1875	AssertCompileSizeAlignment(PGMMAPSET, 8);
1876	/** Pointer to the mapping cache set. */
1877	typedef PGMMAPSET *PPGMMAPSET;
1878
1879	/** PGMMAPSET::cEntries value for a closed set. */
1880	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1881
1882	/** Hash function for aiHashTable. */
1883	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1884
1885
1886	/** @name Context neutral page mapper TLB.
1887	*
1888	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1889	* code is writting in a kind of context neutral way. Time will show whether
1890	* this actually makes sense or not...
1891	*
1892	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1893	* context ends up using a global mapping cache on some platforms
1894	* (darwin).
1895	*
1896	* @{ */
1897	/** @typedef PPGMPAGEMAPTLB
1898	* The page mapper TLB pointer type for the current context. */
1899	/** @typedef PPGMPAGEMAPTLB
1900	* The page mapper TLB entry pointer type for the current context. */
1901	/** @typedef PPGMPAGEMAPTLB
1902	* The page mapper TLB entry pointer pointer type for the current context. */
1903	/** @def PGM_PAGEMAPTLB_ENTRIES
1904	* The number of TLB entries in the page mapper TLB for the current context. */
1905	/** @def PGM_PAGEMAPTLB_IDX
1906	* Calculate the TLB index for a guest physical address.
1907	* @returns The TLB index.
1908	* @param GCPhys The guest physical address. */
1909	/** @typedef PPGMPAGEMAP
1910	* Pointer to a page mapper unit for current context. */
1911	/** @typedef PPPGMPAGEMAP
1912	* Pointer to a page mapper unit pointer for current context. */
1913	#ifdef IN_RC
1914	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1915	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1916	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1917	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1918	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1919	typedef void * PPGMPAGEMAP;
1920	typedef void ** PPPGMPAGEMAP;
1921	//#elif IN_RING0
1922	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1923	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1924	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1925	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1926	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1927	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1928	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1929	#else
1930	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1931	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1932	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1933	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1934	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1935	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1936	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1937	#endif
1938	/** @} */
1939
1940
1941	/** @name PGM Pool Indexes.
1942	* Aka. the unique shadow page identifier.
1943	* @{ */
1944	/** NIL page pool IDX. */
1945	#define NIL_PGMPOOL_IDX 0
1946	/** The first normal index. */
1947	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1948	/** Page directory (32-bit root). */
1949	#define PGMPOOL_IDX_PD 1
1950	/** Page Directory Pointer Table (PAE root). */
1951	#define PGMPOOL_IDX_PDPT 2
1952	/** AMD64 CR3 level index.*/
1953	#define PGMPOOL_IDX_AMD64_CR3 3
1954	/** Nested paging root.*/
1955	#define PGMPOOL_IDX_NESTED_ROOT 4
1956	/** The first normal index. */
1957	#define PGMPOOL_IDX_FIRST 5
1958	/** The last valid index. (inclusive, 14 bits) */
1959	#define PGMPOOL_IDX_LAST 0x3fff
1960	/** @} */
1961
1962	/** The NIL index for the parent chain. */
1963	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1964	#define NIL_PGMPOOL_PRESENT_INDEX ((uint16_t)0xffff)
1965
1966	/**
1967	* Node in the chain linking a shadowed page to it's parent (user).
1968	*/
1969	#pragma pack(1)
1970	typedef struct PGMPOOLUSER
1971	{
1972	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1973	uint16_t iNext;
1974	/** The user page index. */
1975	uint16_t iUser;
1976	/** Index into the user table. */
1977	uint32_t iUserTable;
1978	} PGMPOOLUSER, *PPGMPOOLUSER;
1979	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1980	#pragma pack()
1981
1982
1983	/** The NIL index for the phys ext chain. */
1984	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1985	/** The NIL pte index for a phys ext chain slot. */
1986	#define NIL_PGMPOOL_PHYSEXT_IDX_PTE ((uint16_t)0xffff)
1987
1988	/**
1989	* Node in the chain of physical cross reference extents.
1990	* @todo Calling this an 'extent' is not quite right, find a better name.
1991	* @todo find out the optimal size of the aidx array
1992	*/
1993	#pragma pack(1)
1994	typedef struct PGMPOOLPHYSEXT
1995	{
1996	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1997	uint16_t iNext;
1998	/** Alignment. */
1999	uint16_t u16Align;
2000	/** The user page index. */
2001	uint16_t aidx[3];
2002	/** The page table index or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown. */
2003	uint16_t apte[3];
2004	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
2005	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
2006	#pragma pack()
2007
2008
2009	/**
2010	* The kind of page that's being shadowed.
2011	*/
2012	typedef enum PGMPOOLKIND
2013	{
2014	/** The virtual invalid 0 entry. */
2015	PGMPOOLKIND_INVALID = 0,
2016	/** The entry is free (=unused). */
2017	PGMPOOLKIND_FREE,
2018
2019	/** Shw: 32-bit page table; Gst: no paging */
2020	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
2021	/** Shw: 32-bit page table; Gst: 32-bit page table. */
2022	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
2023	/** Shw: 32-bit page table; Gst: 4MB page. */
2024	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
2025	/** Shw: PAE page table; Gst: no paging */
2026	PGMPOOLKIND_PAE_PT_FOR_PHYS,
2027	/** Shw: PAE page table; Gst: 32-bit page table. */
2028	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
2029	/** Shw: PAE page table; Gst: Half of a 4MB page. */
2030	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
2031	/** Shw: PAE page table; Gst: PAE page table. */
2032	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
2033	/** Shw: PAE page table; Gst: 2MB page. */
2034	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
2035
2036	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
2037	PGMPOOLKIND_32BIT_PD,
2038	/** Shw: 32-bit page directory. Gst: no paging. */
2039	PGMPOOLKIND_32BIT_PD_PHYS,
2040	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
2041	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
2042	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
2043	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
2044	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
2045	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
2046	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
2047	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
2048	/** Shw: PAE page directory; Gst: PAE page directory. */
2049	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
2050	/** Shw: PAE page directory; Gst: no paging. Note: +NP. */
2051	PGMPOOLKIND_PAE_PD_PHYS,
2052
2053	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
2054	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
2055	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
2056	PGMPOOLKIND_PAE_PDPT,
2057	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
2058	PGMPOOLKIND_PAE_PDPT_PHYS,
2059
2060	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
2061	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
2062	/** Shw: 64-bit page directory pointer table; Gst: no paging */
2063	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
2064	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
2065	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
2066	/** Shw: 64-bit page directory table; Gst: no paging */
2067	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
2068
2069	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
2070	PGMPOOLKIND_64BIT_PML4,
2071
2072	/** Shw: EPT page directory pointer table; Gst: no paging */
2073	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
2074	/** Shw: EPT page directory table; Gst: no paging */
2075	PGMPOOLKIND_EPT_PD_FOR_PHYS,
2076	/** Shw: EPT page table; Gst: no paging */
2077	PGMPOOLKIND_EPT_PT_FOR_PHYS,
2078
2079	/** Shw: Root Nested paging table. */
2080	PGMPOOLKIND_ROOT_NESTED,
2081
2082	/** The last valid entry. */
2083	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
2084	} PGMPOOLKIND;
2085
2086	/**
2087	* The access attributes of the page; only applies to big pages.
2088	*/
2089	typedef enum
2090	{
2091	PGMPOOLACCESS_DONTCARE = 0,
2092	PGMPOOLACCESS_USER_RW,
2093	PGMPOOLACCESS_USER_R,
2094	PGMPOOLACCESS_USER_RW_NX,
2095	PGMPOOLACCESS_USER_R_NX,
2096	PGMPOOLACCESS_SUPERVISOR_RW,
2097	PGMPOOLACCESS_SUPERVISOR_R,
2098	PGMPOOLACCESS_SUPERVISOR_RW_NX,
2099	PGMPOOLACCESS_SUPERVISOR_R_NX
2100	} PGMPOOLACCESS;
2101
2102	/**
2103	* The tracking data for a page in the pool.
2104	*/
2105	typedef struct PGMPOOLPAGE
2106	{
2107	/** AVL node code with the (R3) physical address of this page. */
2108	AVLOHCPHYSNODECORE Core;
2109	/** Pointer to the R3 mapping of the page. */
2110	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2111	R3PTRTYPE(void *) pvPageR3;
2112	#else
2113	R3R0PTRTYPE(void *) pvPageR3;
2114	#endif
2115	/** The guest physical address. */
2116	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
2117	uint32_t Alignment0;
2118	#endif
2119	RTGCPHYS GCPhys;
2120
2121	/** Access handler statistics to determine whether the guest is (re)initializing a page table. */
2122	RTGCPTR pvLastAccessHandlerRip;
2123	RTGCPTR pvLastAccessHandlerFault;
2124	uint64_t cLastAccessHandlerCount;
2125
2126	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
2127	uint8_t enmKind;
2128	/** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
2129	uint8_t enmAccess;
2130	/** The index of this page. */
2131	uint16_t idx;
2132	/** The next entry in the list this page currently resides in.
2133	* It's either in the free list or in the GCPhys hash. */
2134	uint16_t iNext;
2135	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
2136	uint16_t iUserHead;
2137	/** The number of present entries. */
2138	uint16_t cPresent;
2139	/** The first entry in the table which is present. */
2140	uint16_t iFirstPresent;
2141	/** The number of modifications to the monitored page. */
2142	uint16_t cModifications;
2143	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
2144	uint16_t iModifiedNext;
2145	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
2146	uint16_t iModifiedPrev;
2147	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
2148	uint16_t iMonitoredNext;
2149	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
2150	uint16_t iMonitoredPrev;
2151	/** The next page in the age list. */
2152	uint16_t iAgeNext;
2153	/** The previous page in the age list. */
2154	uint16_t iAgePrev;
2155	/** Used to indicate that the page is zeroed. */
2156	bool fZeroed;
2157	/** Used to indicate that a PT has non-global entries. */
2158	bool fSeenNonGlobal;
2159	/** Used to indicate that we're monitoring writes to the guest page. */
2160	bool fMonitored;
2161	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
2162	* (All pages are in the age list.) */
2163	bool fCached;
2164	/** This is used by the R3 access handlers when invoked by an async thread.
2165	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
2166	bool volatile fReusedFlushPending;
2167	/** Used to mark the page as dirty (write monitoring is temporarily
2168	* off). */
2169	bool fDirty;
2170
2171	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
2172	uint32_t cLocked;
2173	uint32_t idxDirty;
2174	RTGCPTR pvDirtyFault;
2175	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
2176	/** Pointer to a const pool page. */
2177	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
2178
2179
2180	/** The hash table size. */
2181	# define PGMPOOL_HASH_SIZE 0x40
2182	/** The hash function. */
2183	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
2184
2185
2186	/**
2187	* The shadow page pool instance data.
2188	*
2189	* It's all one big allocation made at init time, except for the
2190	* pages that is. The user nodes follows immediately after the
2191	* page structures.
2192	*/
2193	typedef struct PGMPOOL
2194	{
2195	/** The VM handle - R3 Ptr. */
2196	PVMR3 pVMR3;
2197	/** The VM handle - R0 Ptr. */
2198	PVMR0 pVMR0;
2199	/** The VM handle - RC Ptr. */
2200	PVMRC pVMRC;
2201	/** The max pool size. This includes the special IDs. */
2202	uint16_t cMaxPages;
2203	/** The current pool size. */
2204	uint16_t cCurPages;
2205	/** The head of the free page list. */
2206	uint16_t iFreeHead;
2207	/* Padding. */
2208	uint16_t u16Padding;
2209	/** Head of the chain of free user nodes. */
2210	uint16_t iUserFreeHead;
2211	/** The number of user nodes we've allocated. */
2212	uint16_t cMaxUsers;
2213	/** The number of present page table entries in the entire pool. */
2214	uint32_t cPresent;
2215	/** Pointer to the array of user nodes - RC pointer. */
2216	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
2217	/** Pointer to the array of user nodes - R3 pointer. */
2218	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
2219	/** Pointer to the array of user nodes - R0 pointer. */
2220	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
2221	/** Head of the chain of free phys ext nodes. */
2222	uint16_t iPhysExtFreeHead;
2223	/** The number of user nodes we've allocated. */
2224	uint16_t cMaxPhysExts;
2225	/** Pointer to the array of physical xref extent - RC pointer. */
2226	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
2227	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
2228	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
2229	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
2230	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
2231	/** Hash table for GCPhys addresses. */
2232	uint16_t aiHash[PGMPOOL_HASH_SIZE];
2233	/** The head of the age list. */
2234	uint16_t iAgeHead;
2235	/** The tail of the age list. */
2236	uint16_t iAgeTail;
2237	/** Set if the cache is enabled. */
2238	bool fCacheEnabled;
2239	/** Alignment padding. */
2240	bool afPadding1[3];
2241	/** Head of the list of modified pages. */
2242	uint16_t iModifiedHead;
2243	/** The current number of modified pages. */
2244	uint16_t cModifiedPages;
2245	/** Access handler, RC. */
2246	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
2247	/** Access handler, R0. */
2248	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
2249	/** Access handler, R3. */
2250	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
2251	/** The access handler description (R3 ptr). */
2252	R3PTRTYPE(const char *) pszAccessHandler;
2253	# if HC_ARCH_BITS == 32
2254	/** Alignment padding. */
2255	uint32_t u32Padding2;
2256	# endif
2257	/* Next available slot. */
2258	uint32_t idxFreeDirtyPage;
2259	/* Number of active dirty pages. */
2260	uint32_t cDirtyPages;
2261	/* Array of current dirty pgm pool page indices. */
2262	struct
2263	{
2264	uint16_t uIdx;
2265	uint16_t Alignment[3];
2266	uint64_t aPage[512];
2267	} aDirtyPages[16];
2268	/** The number of pages currently in use. */
2269	uint16_t cUsedPages;
2270	#ifdef VBOX_WITH_STATISTICS
2271	/** The high water mark for cUsedPages. */
2272	uint16_t cUsedPagesHigh;
2273	uint32_t Alignment1; /*< Align the next member on a 64-bit boundary. /
2274	/** Profiling pgmPoolAlloc(). */
2275	STAMPROFILEADV StatAlloc;
2276	/** Profiling pgmR3PoolClearDoIt(). */
2277	STAMPROFILE StatClearAll;
2278	/** Profiling pgmR3PoolReset(). */
2279	STAMPROFILE StatR3Reset;
2280	/** Profiling pgmPoolFlushPage(). */
2281	STAMPROFILE StatFlushPage;
2282	/** Profiling pgmPoolFree(). */
2283	STAMPROFILE StatFree;
2284	/** Counting explicit flushes by PGMPoolFlushPage(). */
2285	STAMCOUNTER StatForceFlushPage;
2286	/** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
2287	STAMCOUNTER StatForceFlushDirtyPage;
2288	/** Counting flushes for reused pages. */
2289	STAMCOUNTER StatForceFlushReused;
2290	/** Profiling time spent zeroing pages. */
2291	STAMPROFILE StatZeroPage;
2292	/** Profiling of pgmPoolTrackDeref. */
2293	STAMPROFILE StatTrackDeref;
2294	/** Profiling pgmTrackFlushGCPhysPT. */
2295	STAMPROFILE StatTrackFlushGCPhysPT;
2296	/** Profiling pgmTrackFlushGCPhysPTs. */
2297	STAMPROFILE StatTrackFlushGCPhysPTs;
2298	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
2299	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
2300	/** Number of times we've been out of user records. */
2301	STAMCOUNTER StatTrackFreeUpOneUser;
2302	/** Nr of flushed entries. */
2303	STAMCOUNTER StatTrackFlushEntry;
2304	/** Nr of updated entries. */
2305	STAMCOUNTER StatTrackFlushEntryKeep;
2306	/** Profiling deref activity related tracking GC physical pages. */
2307	STAMPROFILE StatTrackDerefGCPhys;
2308	/** Number of linear searches for a HCPhys in the ram ranges. */
2309	STAMCOUNTER StatTrackLinearRamSearches;
2310	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
2311	STAMCOUNTER StamTrackPhysExtAllocFailures;
2312	/** Profiling the RC/R0 access handler. */
2313	STAMPROFILE StatMonitorRZ;
2314	/** Times we've failed interpreting the instruction. */
2315	STAMCOUNTER StatMonitorRZEmulateInstr;
2316	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
2317	STAMPROFILE StatMonitorRZFlushPage;
2318	/* Times we've detected a page table reinit. */
2319	STAMCOUNTER StatMonitorRZFlushReinit;
2320	/** Counting flushes for pages that are modified too often. */
2321	STAMCOUNTER StatMonitorRZFlushModOverflow;
2322	/** Times we've detected fork(). */
2323	STAMCOUNTER StatMonitorRZFork;
2324	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
2325	STAMPROFILE StatMonitorRZHandled;
2326	/** Times we've failed interpreting a patch code instruction. */
2327	STAMCOUNTER StatMonitorRZIntrFailPatch1;
2328	/** Times we've failed interpreting a patch code instruction during flushing. */
2329	STAMCOUNTER StatMonitorRZIntrFailPatch2;
2330	/** The number of times we've seen rep prefixes we can't handle. */
2331	STAMCOUNTER StatMonitorRZRepPrefix;
2332	/** Profiling the REP STOSD cases we've handled. */
2333	STAMPROFILE StatMonitorRZRepStosd;
2334	/** Nr of handled PT faults. */
2335	STAMCOUNTER StatMonitorRZFaultPT;
2336	/** Nr of handled PD faults. */
2337	STAMCOUNTER StatMonitorRZFaultPD;
2338	/** Nr of handled PDPT faults. */
2339	STAMCOUNTER StatMonitorRZFaultPDPT;
2340	/** Nr of handled PML4 faults. */
2341	STAMCOUNTER StatMonitorRZFaultPML4;
2342
2343	/** Profiling the R3 access handler. */
2344	STAMPROFILE StatMonitorR3;
2345	/** Times we've failed interpreting the instruction. */
2346	STAMCOUNTER StatMonitorR3EmulateInstr;
2347	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
2348	STAMPROFILE StatMonitorR3FlushPage;
2349	/* Times we've detected a page table reinit. */
2350	STAMCOUNTER StatMonitorR3FlushReinit;
2351	/** Counting flushes for pages that are modified too often. */
2352	STAMCOUNTER StatMonitorR3FlushModOverflow;
2353	/** Times we've detected fork(). */
2354	STAMCOUNTER StatMonitorR3Fork;
2355	/** Profiling the R3 access we've handled (except REP STOSD). */
2356	STAMPROFILE StatMonitorR3Handled;
2357	/** The number of times we've seen rep prefixes we can't handle. */
2358	STAMCOUNTER StatMonitorR3RepPrefix;
2359	/** Profiling the REP STOSD cases we've handled. */
2360	STAMPROFILE StatMonitorR3RepStosd;
2361	/** Nr of handled PT faults. */
2362	STAMCOUNTER StatMonitorR3FaultPT;
2363	/** Nr of handled PD faults. */
2364	STAMCOUNTER StatMonitorR3FaultPD;
2365	/** Nr of handled PDPT faults. */
2366	STAMCOUNTER StatMonitorR3FaultPDPT;
2367	/** Nr of handled PML4 faults. */
2368	STAMCOUNTER StatMonitorR3FaultPML4;
2369	/** The number of times we're called in an async thread an need to flush. */
2370	STAMCOUNTER StatMonitorR3Async;
2371	/** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
2372	STAMCOUNTER StatResetDirtyPages;
2373	/** Times we've called pgmPoolAddDirtyPage. */
2374	STAMCOUNTER StatDirtyPage;
2375	/** Times we've had to flush duplicates for dirty page management. */
2376	STAMCOUNTER StatDirtyPageDupFlush;
2377	/** Times we've had to flush because of overflow. */
2378	STAMCOUNTER StatDirtyPageOverFlowFlush;
2379
2380	/** The high water mark for cModifiedPages. */
2381	uint16_t cModifiedPagesHigh;
2382	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundary. /
2383
2384	/** The number of cache hits. */
2385	STAMCOUNTER StatCacheHits;
2386	/** The number of cache misses. */
2387	STAMCOUNTER StatCacheMisses;
2388	/** The number of times we've got a conflict of 'kind' in the cache. */
2389	STAMCOUNTER StatCacheKindMismatches;
2390	/** Number of times we've been out of pages. */
2391	STAMCOUNTER StatCacheFreeUpOne;
2392	/** The number of cacheable allocations. */
2393	STAMCOUNTER StatCacheCacheable;
2394	/** The number of uncacheable allocations. */
2395	STAMCOUNTER StatCacheUncacheable;
2396	#else
2397	uint32_t Alignment3; /*< Align the next member on a 64-bit boundary. /
2398	#endif
2399	/** The AVL tree for looking up a page by its HC physical address. */
2400	AVLOHCPHYSTREE HCPhysTree;
2401	uint32_t Alignment4; /*< Align the next member on a 64-bit boundary. /
2402	/** Array of pages. (cMaxPages in length)
2403	* The Id is the index into thist array.
2404	*/
2405	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
2406	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
2407	AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
2408	AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
2409	AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
2410	#ifdef VBOX_WITH_STATISTICS
2411	AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
2412	#endif
2413	AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
2414
2415
2416	/** @def PGMPOOL_PAGE_2_PTR
2417	* Maps a pool page pool into the current context.
2418	*
2419	* @returns VBox status code.
2420	* @param a_pVM The VM handle.
2421	* @param a_pPage The pool page.
2422	*
2423	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2424	* small page window employeed by that function. Be careful.
2425	* @remark There is no need to assert on the result.
2426	*/
2427	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2428	# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage) pgmPoolMapPageInlined((a_pVM), (a_pPage) RTLOG_COMMA_SRC_POS)
2429	#elif defined(VBOX_STRICT)
2430	# define PGMPOOL_PAGE_2_PTR(a_pVM, a_pPage) pgmPoolMapPageStrict(a_pPage)
2431	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE a_pPage)
2432	{
2433	Assert(a_pPage && a_pPage->pvPageR3);
2434	return a_pPage->pvPageR3;
2435	}
2436	#else
2437	# define PGMPOOL_PAGE_2_PTR(pVM, a_pPage) ((a_pPage)->pvPageR3)
2438	#endif
2439
2440
2441	/** @def PGMPOOL_PAGE_2_PTR_V2
2442	* Maps a pool page pool into the current context, taking both VM and VMCPU.
2443	*
2444	* @returns VBox status code.
2445	* @param a_pVM The VM handle.
2446	* @param a_pVCpu The current CPU.
2447	* @param a_pPage The pool page.
2448	*
2449	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2450	* small page window employeed by that function. Be careful.
2451	* @remark There is no need to assert on the result.
2452	*/
2453	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2454	# define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage) pgmPoolMapPageV2Inlined((a_pVM), (a_pVCpu), (a_pPage) RTLOG_COMMA_SRC_POS)
2455	#else
2456	# define PGMPOOL_PAGE_2_PTR_V2(a_pVM, a_pVCpu, a_pPage) PGMPOOL_PAGE_2_PTR((a_pVM), (a_pPage))
2457	#endif
2458
2459
2460	/** @name Per guest page tracking data.
2461	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
2462	* is to use more bits for it and split it up later on. But for now we'll play
2463	* safe and change as little as possible.
2464	*
2465	* The 16-bit word has two parts:
2466	*
2467	* The first 14-bit forms the @a idx field. It is either the index of a page in
2468	* the shadow page pool, or and index into the extent list.
2469	*
2470	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
2471	* shadow page pool references to the page. If cRefs equals
2472	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
2473	* (misnomer) table and not the shadow page pool.
2474	*
2475	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
2476	* the 16-bit word.
2477	*
2478	* @{ */
2479	/** The shift count for getting to the cRefs part. */
2480	#define PGMPOOL_TD_CREFS_SHIFT 14
2481	/** The mask applied after shifting the tracking data down by
2482	* PGMPOOL_TD_CREFS_SHIFT. */
2483	#define PGMPOOL_TD_CREFS_MASK 0x3
2484	/** The cRefs value used to indicate that the idx is the head of a
2485	* physical cross reference list. */
2486	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
2487	/** The shift used to get idx. */
2488	#define PGMPOOL_TD_IDX_SHIFT 0
2489	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
2490	#define PGMPOOL_TD_IDX_MASK 0x3fff
2491	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
2492	* simply too many mappings of this page. */
2493	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
2494
2495	/** @def PGMPOOL_TD_MAKE
2496	* Makes a 16-bit tracking data word.
2497	*
2498	* @returns tracking data.
2499	* @param cRefs The @a cRefs field. Must be within bounds!
2500	* @param idx The @a idx field. Must also be within bounds! */
2501	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
2502
2503	/** @def PGMPOOL_TD_GET_CREFS
2504	* Get the @a cRefs field from a tracking data word.
2505	*
2506	* @returns The @a cRefs field
2507	* @param u16 The tracking data word.
2508	* @remarks This will only return 1 or PGMPOOL_TD_CREFS_PHYSEXT for a
2509	* non-zero @a u16. */
2510	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
2511
2512	/** @def PGMPOOL_TD_GET_IDX
2513	* Get the @a idx field from a tracking data word.
2514	*
2515	* @returns The @a idx field
2516	* @param u16 The tracking data word. */
2517	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
2518	/** @} */
2519
2520
2521	/**
2522	* Trees are using self relative offsets as pointers.
2523	* So, all its data, including the root pointer, must be in the heap for HC and GC
2524	* to have the same layout.
2525	*/
2526	typedef struct PGMTREES
2527	{
2528	/** Physical access handlers (AVL range+offsetptr tree). */
2529	AVLROGCPHYSTREE PhysHandlers;
2530	/** Virtual access handlers (AVL range + GC ptr tree). */
2531	AVLROGCPTRTREE VirtHandlers;
2532	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
2533	AVLROGCPHYSTREE PhysToVirtHandlers;
2534	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
2535	AVLROGCPTRTREE HyperVirtHandlers;
2536	} PGMTREES;
2537	/** Pointer to PGM trees. */
2538	typedef PGMTREES *PPGMTREES;
2539
2540
2541	/**
2542	* Page fault guest state for the AMD64 paging mode.
2543	*/
2544	typedef struct PGMPTWALKCORE
2545	{
2546	/** The guest virtual address that is being resolved by the walk
2547	* (input). */
2548	RTGCPTR GCPtr;
2549
2550	/** The guest physical address that is the result of the walk.
2551	* @remarks only valid if fSucceeded is set. */
2552	RTGCPHYS GCPhys;
2553
2554	/** Set if the walk succeeded, i.d. GCPhys is valid. */
2555	bool fSucceeded;
2556	/** The level problem arrised at.
2557	* PTE is level 1, PDE is level 2, PDPE is level 3, PML4 is level 4, CR3 is
2558	* level 8. This is 0 on success. */
2559	uint8_t uLevel;
2560	/** Set if the page isn't present. */
2561	bool fNotPresent;
2562	/** Encountered a bad physical address. */
2563	bool fBadPhysAddr;
2564	/** Set if there was reserved bit violations. */
2565	bool fRsvdError;
2566	/** Set if it involves a big page (2/4 MB). */
2567	bool fBigPage;
2568	/** Set if it involves a gigantic page (1 GB). */
2569	bool fGigantPage;
2570	/** The effect X86_PTE_US flag for the address. */
2571	bool fEffectiveUS;
2572	/** The effect X86_PTE_RW flag for the address. */
2573	bool fEffectiveRW;
2574	/** The effect X86_PTE_NX flag for the address. */
2575	bool fEffectiveNX;
2576	} PGMPTWALKCORE;
2577
2578
2579	/**
2580	* Guest page table walk for the AMD64 mode.
2581	*/
2582	typedef struct PGMPTWALKGSTAMD64
2583	{
2584	/** The common core. */
2585	PGMPTWALKCORE Core;
2586
2587	PX86PML4 pPml4;
2588	PX86PML4E pPml4e;
2589	X86PML4E Pml4e;
2590
2591	PX86PDPT pPdpt;
2592	PX86PDPE pPdpe;
2593	X86PDPE Pdpe;
2594
2595	PX86PDPAE pPd;
2596	PX86PDEPAE pPde;
2597	X86PDEPAE Pde;
2598
2599	PX86PTPAE pPt;
2600	PX86PTEPAE pPte;
2601	X86PTEPAE Pte;
2602	} PGMPTWALKGSTAMD64;
2603	/** Pointer to a AMD64 guest page table walk. */
2604	typedef PGMPTWALKGSTAMD64 *PPGMPTWALKGSTAMD64;
2605	/** Pointer to a const AMD64 guest page table walk. */
2606	typedef PGMPTWALKGSTAMD64 const *PCPGMPTWALKGSTAMD64;
2607
2608	/**
2609	* Guest page table walk for the PAE mode.
2610	*/
2611	typedef struct PGMPTWALKGSTPAE
2612	{
2613	/** The common core. */
2614	PGMPTWALKCORE Core;
2615
2616	PX86PDPT pPdpt;
2617	PX86PDPE pPdpe;
2618	X86PDPE Pdpe;
2619
2620	PX86PDPAE pPd;
2621	PX86PDEPAE pPde;
2622	X86PDEPAE Pde;
2623
2624	PX86PTPAE pPt;
2625	PX86PTEPAE pPte;
2626	X86PTEPAE Pte;
2627	} PGMPTWALKGSTPAE;
2628	/** Pointer to a PAE guest page table walk. */
2629	typedef PGMPTWALKGSTPAE *PPGMPTWALKGSTPAE;
2630	/** Pointer to a const AMD64 guest page table walk. */
2631	typedef PGMPTWALKGSTPAE const *PCPGMPTWALKGSTPAE;
2632
2633	/**
2634	* Guest page table walk for the 32-bit mode.
2635	*/
2636	typedef struct PGMPTWALKGST32BIT
2637	{
2638	/** The common core. */
2639	PGMPTWALKCORE Core;
2640
2641	PX86PD pPd;
2642	PX86PDE pPde;
2643	X86PDE Pde;
2644
2645	PX86PT pPt;
2646	PX86PTE pPte;
2647	X86PTE Pte;
2648	} PGMPTWALKGST32BIT;
2649	/** Pointer to a 32-bit guest page table walk. */
2650	typedef PGMPTWALKGST32BIT *PPGMPTWALKGST32BIT;
2651	/** Pointer to a const 32-bit guest page table walk. */
2652	typedef PGMPTWALKGST32BIT const *PCPGMPTWALKGST32BIT;
2653
2654
2655	/** @name Paging mode macros
2656	* @{
2657	*/
2658	#ifdef IN_RC
2659	# define PGM_CTX(a,b) a##RC##b
2660	# define PGM_CTX_STR(a,b) a "GC" b
2661	# define PGM_CTX_DECL(type) VMMRCDECL(type)
2662	#else
2663	# ifdef IN_RING3
2664	# define PGM_CTX(a,b) a##R3##b
2665	# define PGM_CTX_STR(a,b) a "R3" b
2666	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
2667	# else
2668	# define PGM_CTX(a,b) a##R0##b
2669	# define PGM_CTX_STR(a,b) a "R0" b
2670	# define PGM_CTX_DECL(type) VMMDECL(type)
2671	# endif
2672	#endif
2673
2674	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
2675	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
2676	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
2677	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
2678	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
2679	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
2680	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
2681	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
2682	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
2683	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
2684	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
2685	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
2686	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
2687	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
2688	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
2689	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
2690	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
2691
2692	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
2693	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
2694	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
2695	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
2696	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
2697	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
2698	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
2699	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
2700	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
2701	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
2702	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
2703	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2704	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2705	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2706	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2707	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2708	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2709
2710	/* Shw_Gst */
2711	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2712	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2713	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2714	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2715	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2716	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2717	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2718	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2719	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2720	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2721	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2722	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2723	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2724	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2725	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2726	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2727	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2728	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2729	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2730
2731	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2732	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2733	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2734	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2735	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2736	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2737	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2738	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2739	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2740	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2741	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2742	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2743	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2744	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2745	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2746	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2747	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2748	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2749	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2750	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2751	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2752	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2753	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2754	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2755	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2756	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2757	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2758	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2759	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2760	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2761	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2762	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2763	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2764	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2765	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2766	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2767	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2768
2769	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2770	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2771	/** @} */
2772
2773	/**
2774	* Data for each paging mode.
2775	*/
2776	typedef struct PGMMODEDATA
2777	{
2778	/** The guest mode type. */
2779	uint32_t uGstType;
2780	/** The shadow mode type. */
2781	uint32_t uShwType;
2782
2783	/** @name Function pointers for Shadow paging.
2784	* @{
2785	*/
2786	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2787	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2788	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2789	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
2790
2791	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2792	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
2793
2794	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2795	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
2796	/** @} */
2797
2798	/** @name Function pointers for Guest paging.
2799	* @{
2800	*/
2801	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2802	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2803	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2804	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2805	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2806	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2807	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2808	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2809	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2810	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2811	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2812	/** @} */
2813
2814	/** @name Function pointers for Both Shadow and Guest paging.
2815	* @{
2816	*/
2817	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2818	/* no pfnR3BthTrap0eHandler */
2819	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2820	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2821	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2822	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2823	#ifdef VBOX_STRICT
2824	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2825	#endif
2826	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2827	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2828
2829	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
2830	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2831	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2832	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2833	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2834	#ifdef VBOX_STRICT
2835	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2836	#endif
2837	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2838	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2839
2840	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
2841	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2842	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2843	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2844	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2845	#ifdef VBOX_STRICT
2846	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2847	#endif
2848	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2849	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2850	/** @} */
2851	} PGMMODEDATA, *PPGMMODEDATA;
2852
2853
2854	#ifdef VBOX_WITH_STATISTICS
2855	/**
2856	* PGM statistics.
2857	*
2858	* These lives on the heap when compiled in as they would otherwise waste
2859	* unnecessary space in release builds.
2860	*/
2861	typedef struct PGMSTATS
2862	{
2863	/* R3 only: */
2864	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2865	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2866
2867	/* R3+RZ */
2868	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2869	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2870	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2871	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2872	STAMCOUNTER StatPageMapTlbFlushes; /*< ALL: Ring-3/0 page mapper TLB flushes. /
2873	STAMCOUNTER StatPageMapTlbFlushEntry; /*< ALL: Ring-3/0 page mapper TLB flushes. /
2874	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2875	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2876	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2877	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2878	STAMCOUNTER StatRZRamRangeTlbHits; /*< RC/R0: RAM range TLB hits. /
2879	STAMCOUNTER StatRZRamRangeTlbMisses; /*< RC/R0: RAM range TLB misses. /
2880	STAMCOUNTER StatR3RamRangeTlbHits; /*< R3: RAM range TLB hits. /
2881	STAMCOUNTER StatR3RamRangeTlbMisses; /*< R3: RAM range TLB misses. /
2882	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2883	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2884	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2885	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2886	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2887	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2888	STAMCOUNTER StatR3PhysHandlerLookupHits; /*< R3: Number of cache hits when looking up physical handlers. /
2889	STAMCOUNTER StatR3PhysHandlerLookupMisses; /*< R3: Number of cache misses when looking up physical handlers. /
2890	STAMCOUNTER StatRZPhysHandlerLookupHits; /*< RC/R0: Number of cache hits when lookup up physical handlers. /
2891	STAMCOUNTER StatRZPhysHandlerLookupMisses; /*< RC/R0: Number of cache misses when looking up physical handlers /
2892	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2893	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2894	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2895	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2896	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2897	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2898	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2899	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2900
2901	/* RC only: */
2902	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2903	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2904
2905	STAMCOUNTER StatRZPhysRead;
2906	STAMCOUNTER StatRZPhysReadBytes;
2907	STAMCOUNTER StatRZPhysWrite;
2908	STAMCOUNTER StatRZPhysWriteBytes;
2909	STAMCOUNTER StatR3PhysRead;
2910	STAMCOUNTER StatR3PhysReadBytes;
2911	STAMCOUNTER StatR3PhysWrite;
2912	STAMCOUNTER StatR3PhysWriteBytes;
2913	STAMCOUNTER StatRCPhysRead;
2914	STAMCOUNTER StatRCPhysReadBytes;
2915	STAMCOUNTER StatRCPhysWrite;
2916	STAMCOUNTER StatRCPhysWriteBytes;
2917
2918	STAMCOUNTER StatRZPhysSimpleRead;
2919	STAMCOUNTER StatRZPhysSimpleReadBytes;
2920	STAMCOUNTER StatRZPhysSimpleWrite;
2921	STAMCOUNTER StatRZPhysSimpleWriteBytes;
2922	STAMCOUNTER StatR3PhysSimpleRead;
2923	STAMCOUNTER StatR3PhysSimpleReadBytes;
2924	STAMCOUNTER StatR3PhysSimpleWrite;
2925	STAMCOUNTER StatR3PhysSimpleWriteBytes;
2926	STAMCOUNTER StatRCPhysSimpleRead;
2927	STAMCOUNTER StatRCPhysSimpleReadBytes;
2928	STAMCOUNTER StatRCPhysSimpleWrite;
2929	STAMCOUNTER StatRCPhysSimpleWriteBytes;
2930
2931	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2932	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2933	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2934	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2935	STAMCOUNTER StatTrackNoExtentsLeft; /*< The number of times the extent list was exhausted. /
2936	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2937	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2938
2939	/** Time spent by the host OS for large page allocation. */
2940	STAMPROFILE StatAllocLargePage;
2941	/** Time spent clearing the newly allocated large pages. */
2942	STAMPROFILE StatClearLargePage;
2943	/** The number of times allocating a large pages takes more than the allowed period. */
2944	STAMCOUNTER StatLargePageOverflow;
2945	/** pgmPhysIsValidLargePage profiling - R3 */
2946	STAMPROFILE StatR3IsValidLargePage;
2947	/** pgmPhysIsValidLargePage profiling - RZ*/
2948	STAMPROFILE StatRZIsValidLargePage;
2949
2950	STAMPROFILE StatChunkAging;
2951	STAMPROFILE StatChunkFindCandidate;
2952	STAMPROFILE StatChunkUnmap;
2953	STAMPROFILE StatChunkMap;
2954	} PGMSTATS;
2955	#endif /* VBOX_WITH_STATISTICS */
2956
2957
2958	/**
2959	* Converts a PGM pointer into a VM pointer.
2960	* @returns Pointer to the VM structure the PGM is part of.
2961	* @param pPGM Pointer to PGM instance data.
2962	*/
2963	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2964
2965	/**
2966	* PGM Data (part of VM)
2967	*/
2968	typedef struct PGM
2969	{
2970	/** Offset to the VM structure. */
2971	int32_t offVM;
2972	/** Offset of the PGMCPU structure relative to VMCPU. */
2973	int32_t offVCpuPGM;
2974
2975	/** @cfgm{RamPreAlloc, boolean, false}
2976	* Indicates whether the base RAM should all be allocated before starting
2977	* the VM (default), or if it should be allocated when first written to.
2978	*/
2979	bool fRamPreAlloc;
2980	/** Indicates whether write monitoring is currently in use.
2981	* This is used to prevent conflicts between live saving and page sharing
2982	* detection. */
2983	bool fPhysWriteMonitoringEngaged;
2984	/** Set if the CPU has less than 52-bit physical address width.
2985	* This is used */
2986	bool fLessThan52PhysicalAddressBits;
2987	/** Set when nested paging is active.
2988	* This is meant to save calls to HWACCMIsNestedPagingActive and let the
2989	* compilers optimize the code better. Whether we use nested paging or
2990	* not is something we find out during VMM initialization and we won't
2991	* change this later on. */
2992	bool fNestedPaging;
2993	/** The host paging mode. (This is what SUPLib reports.) */
2994	SUPPAGINGMODE enmHostMode;
2995	/** We're not in a state which permits writes to guest memory.
2996	* (Only used in strict builds.) */
2997	bool fNoMorePhysWrites;
2998	/** Set if PCI passthrough is enabled. */
2999	bool fPciPassthrough;
3000	/** Alignment padding that makes the next member start on a 8 byte boundary. */
3001	bool afAlignment1[2];
3002
3003	/** Indicates that PGMR3FinalizeMappings has been called and that further
3004	* PGMR3MapIntermediate calls will be rejected. */
3005	bool fFinalizedMappings;
3006	/** If set no conflict checks are required. */
3007	bool fMappingsFixed;
3008	/** If set if restored as fixed but we were unable to re-fixate at the old
3009	* location because of room or address incompatibilities. */
3010	bool fMappingsFixedRestored;
3011	/** If set, then no mappings are put into the shadow page table.
3012	* Use pgmMapAreMappingsEnabled() instead of direct access. */
3013	bool fMappingsDisabled;
3014	/** Size of fixed mapping.
3015	* This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
3016	uint32_t cbMappingFixed;
3017	/** Generation ID for the RAM ranges. This member is incremented everytime
3018	* a RAM range is linked or unlinked. */
3019	uint32_t volatile idRamRangesGen;
3020
3021	/** Base address (GC) of fixed mapping.
3022	* This is valid if either fMappingsFixed or fMappingsFixedRestored is set. */
3023	RTGCPTR GCPtrMappingFixed;
3024	/** The address of the previous RAM range mapping. */
3025	RTGCPTR GCPtrPrevRamRangeMapping;
3026
3027	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
3028	RTGCPHYS GCPhys4MBPSEMask;
3029	/** Mask containing the invalid bits of a guest physical address.
3030	* @remarks this does not stop at bit 52. */
3031	RTGCPHYS GCPhysInvAddrMask;
3032
3033
3034	/** RAM range TLB for R3. */
3035	R3PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR3[PGM_RAMRANGE_TLB_ENTRIES];
3036	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
3037	* This is sorted by physical address and contains no overlapping ranges. */
3038	R3PTRTYPE(PPGMRAMRANGE) pRamRangesXR3;
3039	/** Root of the RAM range search tree for ring-3. */
3040	R3PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR3;
3041	/** PGM offset based trees - R3 Ptr. */
3042	R3PTRTYPE(PPGMTREES) pTreesR3;
3043	/** Caching the last physical handler we looked up in R3. */
3044	R3PTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerR3;
3045	/** Shadow Page Pool - R3 Ptr. */
3046	R3PTRTYPE(PPGMPOOL) pPoolR3;
3047	/** Linked list of GC mappings - for HC.
3048	* The list is sorted ascending on address. */
3049	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
3050	/** Pointer to the list of ROM ranges - for R3.
3051	* This is sorted by physical address and contains no overlapping ranges. */
3052	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
3053	/** Pointer to the list of MMIO2 ranges - for R3.
3054	* Registration order. */
3055	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
3056	/** Pointer to SHW+GST mode data (function pointers).
3057	* The index into this table is made up from */
3058	R3PTRTYPE(PPGMMODEDATA) paModeData;
3059	RTR3PTR R3PtrAlignment0;
3060
3061	/** RAM range TLB for R0. */
3062	R0PTRTYPE(PPGMRAMRANGE) apRamRangesTlbR0[PGM_RAMRANGE_TLB_ENTRIES];
3063	/** R0 pointer corresponding to PGM::pRamRangesXR3. */
3064	R0PTRTYPE(PPGMRAMRANGE) pRamRangesXR0;
3065	/** Root of the RAM range search tree for ring-0. */
3066	R0PTRTYPE(PPGMRAMRANGE) pRamRangeTreeR0;
3067	/** PGM offset based trees - R0 Ptr. */
3068	R0PTRTYPE(PPGMTREES) pTreesR0;
3069	/** Caching the last physical handler we looked up in R0. */
3070	R0PTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerR0;
3071	/** Shadow Page Pool - R0 Ptr. */
3072	R0PTRTYPE(PPGMPOOL) pPoolR0;
3073	/** Linked list of GC mappings - for R0.
3074	* The list is sorted ascending on address. */
3075	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
3076	/** R0 pointer corresponding to PGM::pRomRangesR3. */
3077	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
3078	RTR0PTR R0PtrAlignment0;
3079
3080
3081	/** RAM range TLB for RC. */
3082	RCPTRTYPE(PPGMRAMRANGE) apRamRangesTlbRC[PGM_RAMRANGE_TLB_ENTRIES];
3083	/** RC pointer corresponding to PGM::pRamRangesXR3. */
3084	RCPTRTYPE(PPGMRAMRANGE) pRamRangesXRC;
3085	/** Root of the RAM range search tree for raw-mode context. */
3086	RCPTRTYPE(PPGMRAMRANGE) pRamRangeTreeRC;
3087	/** PGM offset based trees - RC Ptr. */
3088	RCPTRTYPE(PPGMTREES) pTreesRC;
3089	/** Caching the last physical handler we looked up in RC. */
3090	RCPTRTYPE(PPGMPHYSHANDLER) pLastPhysHandlerRC;
3091	/** Shadow Page Pool - RC Ptr. */
3092	RCPTRTYPE(PPGMPOOL) pPoolRC;
3093	/** Linked list of GC mappings - for RC.
3094	* The list is sorted ascending on address. */
3095	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
3096	/** RC pointer corresponding to PGM::pRomRangesR3. */
3097	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
3098	RTRCPTR RCPtrAlignment0;
3099	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
3100	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
3101	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
3102	RCPTRTYPE(PPGMSHWPTEPAE) paDynPageMapPaePTEsGC;
3103
3104
3105	/** Pointer to the 5 page CR3 content mapping.
3106	* The first page is always the CR3 (in some form) while the 4 other pages
3107	* are used of the PDs in PAE mode. */
3108	RTGCPTR GCPtrCR3Mapping;
3109
3110	/** @name Intermediate Context
3111	* @{ */
3112	/** Pointer to the intermediate page directory - Normal. */
3113	R3PTRTYPE(PX86PD) pInterPD;
3114	/** Pointer to the intermediate page tables - Normal.
3115	* There are two page tables, one for the identity mapping and one for
3116	* the host context mapping (of the core code). */
3117	R3PTRTYPE(PX86PT) apInterPTs[2];
3118	/** Pointer to the intermediate page tables - PAE. */
3119	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
3120	/** Pointer to the intermediate page directory - PAE. */
3121	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
3122	/** Pointer to the intermediate page directory - PAE. */
3123	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
3124	/** Pointer to the intermediate page-map level 4 - AMD64. */
3125	R3PTRTYPE(PX86PML4) pInterPaePML4;
3126	/** Pointer to the intermediate page directory - AMD64. */
3127	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
3128	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
3129	RTHCPHYS HCPhysInterPD;
3130	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
3131	RTHCPHYS HCPhysInterPaePDPT;
3132	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
3133	RTHCPHYS HCPhysInterPaePML4;
3134	/** @} */
3135
3136	/** Base address of the dynamic page mapping area.
3137	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
3138	*
3139	* @todo The plan of keeping PGMRCDYNMAP private to PGMRZDynMap.cpp didn't
3140	* work out. Some cleaning up of the initialization that would
3141	* remove this memory is yet to be done...
3142	*/
3143	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
3144	/** The address of the raw-mode context mapping cache. */
3145	RCPTRTYPE(PPGMRCDYNMAP) pRCDynMap;
3146	/** The address of the ring-0 mapping cache if we're making use of it. */
3147	RTR0PTR pvR0DynMapUsed;
3148
3149	/** Hack: Number of deprecated page mapping locks taken by the current lock
3150	* owner via pgmPhysGCPhys2CCPtrInternalDepr. */
3151	uint32_t cDeprecatedPageLocks;
3152	#if HC_ARCH_BITS == 64
3153	/** Alignment padding. */
3154	uint32_t u32Alignment2;
3155	#endif
3156
3157
3158	/** PGM critical section.
3159	* This protects the physical & virtual access handlers, ram ranges,
3160	* and the page flag updating (some of it anyway).
3161	*/
3162	PDMCRITSECT CritSectX;
3163
3164	/**
3165	* Data associated with managing the ring-3 mappings of the allocation chunks.
3166	*/
3167	struct
3168	{
3169	/** The chunk tree, ordered by chunk id. */
3170	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
3171	R3PTRTYPE(PAVLU32NODECORE) pTree;
3172	#else
3173	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
3174	#endif
3175	#if HC_ARCH_BITS == 32
3176	uint32_t u32Alignment;
3177	#endif
3178	/** The chunk mapping TLB. */
3179	PGMCHUNKR3MAPTLB Tlb;
3180	/** The number of mapped chunks. */
3181	uint32_t c;
3182	/** The maximum number of mapped chunks.
3183	* @cfgm PGM/MaxRing3Chunks */
3184	uint32_t cMax;
3185	/** The current time. */
3186	uint32_t iNow;
3187	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
3188	uint32_t AgeingCountdown;
3189	} ChunkR3Map;
3190
3191	/**
3192	* The page mapping TLB for ring-3 and (for the time being) ring-0.
3193	*/
3194	PGMPAGER3MAPTLB PhysTlbHC;
3195
3196	/** @name The zero page.
3197	* @{ */
3198	/** The host physical address of the zero page. */
3199	RTHCPHYS HCPhysZeroPg;
3200	/** The ring-3 mapping of the zero page. */
3201	RTR3PTR pvZeroPgR3;
3202	/** The ring-0 mapping of the zero page. */
3203	RTR0PTR pvZeroPgR0;
3204	/** The GC mapping of the zero page. */
3205	RTRCPTR pvZeroPgRC;
3206	RTRCPTR RCPtrAlignment3;
3207	/** @}*/
3208
3209	/** @name The Invalid MMIO page.
3210	* This page is filled with 0xfeedface.
3211	* @{ */
3212	/** The host physical address of the invalid MMIO page. */
3213	RTHCPHYS HCPhysMmioPg;
3214	/** The host pysical address of the invalid MMIO page plus all invalid
3215	* physical address bits set. This is used to trigger X86_TRAP_PF_RSVD.
3216	* @remarks Check fLessThan52PhysicalAddressBits before use. */
3217	RTHCPHYS HCPhysInvMmioPg;
3218	/** The ring-3 mapping of the invalid MMIO page. */
3219	RTR3PTR pvMmioPgR3;
3220	#if HC_ARCH_BITS == 32
3221	RTR3PTR R3PtrAlignment4;
3222	#endif
3223	/** @} */
3224
3225
3226	/** The number of handy pages. */
3227	uint32_t cHandyPages;
3228
3229	/** The number of large handy pages. */
3230	uint32_t cLargeHandyPages;
3231
3232	/**
3233	* Array of handy pages.
3234	*
3235	* This array is used in a two way communication between pgmPhysAllocPage
3236	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
3237	* an intermediary.
3238	*
3239	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
3240	* (The current size of 32 pages, means 128 KB of handy memory.)
3241	*/
3242	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
3243
3244	/**
3245	* Array of large handy pages. (currently size 1)
3246	*
3247	* This array is used in a two way communication between pgmPhysAllocLargePage
3248	* and GMMR0AllocateLargePage, with PGMR3PhysAllocateLargePage serving as
3249	* an intermediary.
3250	*/
3251	GMMPAGEDESC aLargeHandyPage[1];
3252
3253	/**
3254	* Live save data.
3255	*/
3256	struct
3257	{
3258	/** Per type statistics. */
3259	struct
3260	{
3261	/** The number of ready pages. */
3262	uint32_t cReadyPages;
3263	/** The number of dirty pages. */
3264	uint32_t cDirtyPages;
3265	/** The number of ready zero pages. */
3266	uint32_t cZeroPages;
3267	/** The number of write monitored pages. */
3268	uint32_t cMonitoredPages;
3269	} Rom,
3270	Mmio2,
3271	Ram;
3272	/** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
3273	uint32_t cIgnoredPages;
3274	/** Indicates that a live save operation is active. */
3275	bool fActive;
3276	/** Padding. */
3277	bool afReserved[2];
3278	/** The next history index. */
3279	uint8_t iDirtyPagesHistory;
3280	/** History of the total amount of dirty pages. */
3281	uint32_t acDirtyPagesHistory[64];
3282	/** Short term dirty page average. */
3283	uint32_t cDirtyPagesShort;
3284	/** Long term dirty page average. */
3285	uint32_t cDirtyPagesLong;
3286	/** The number of saved pages. This is used to get some kind of estimate of the
3287	* link speed so we can decide when we're done. It is reset after the first
3288	* 7 passes so the speed estimate doesn't get inflated by the initial set of
3289	* zero pages. */
3290	uint64_t cSavedPages;
3291	/** The nanosecond timestamp when cSavedPages was 0. */
3292	uint64_t uSaveStartNS;
3293	/** Pages per second (for statistics). */
3294	uint32_t cPagesPerSecond;
3295	uint32_t cAlignment;
3296	} LiveSave;
3297
3298	/** @name Error injection.
3299	* @{ */
3300	/** Inject handy page allocation errors pretending we're completely out of
3301	* memory. */
3302	bool volatile fErrInjHandyPages;
3303	/** Padding. */
3304	bool afReserved[3];
3305	/** @} */
3306
3307	/** @name Release Statistics
3308	* @{ */
3309	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero + Pure MMIO.) /
3310	uint32_t cPrivatePages; /*< The number of private pages. /
3311	uint32_t cSharedPages; /*< The number of shared pages. /
3312	uint32_t cReusedSharedPages; /*< The number of reused shared pages. /
3313	uint32_t cZeroPages; /*< The number of zero backed pages. /
3314	uint32_t cPureMmioPages; /*< The number of pure MMIO pages. /
3315	uint32_t cMonitoredPages; /*< The number of write monitored pages. /
3316	uint32_t cWrittenToPages; /*< The number of previously write monitored pages. /
3317	uint32_t cWriteLockedPages; /*< The number of write locked pages. /
3318	uint32_t cReadLockedPages; /*< The number of read locked pages. /
3319	uint32_t cBalloonedPages; /*< The number of ballooned pages. /
3320	uint32_t cMappedChunks; /*< Number of times we mapped a chunk. /
3321	uint32_t cUnmappedChunks; /*< Number of times we unmapped a chunk. /
3322	uint32_t cLargePages; /*< The number of large pages. /
3323	uint32_t cLargePagesDisabled;/*< The number of disabled large pages. /
3324	/* uint32_t aAlignment4[1]; */
3325
3326	/** The number of times we were forced to change the hypervisor region location. */
3327	STAMCOUNTER cRelocations;
3328
3329	STAMCOUNTER StatLargePageReused; /*< The number of large pages we've reused./
3330	STAMCOUNTER StatLargePageRefused; /*< The number of times we couldn't use a large page./
3331	STAMCOUNTER StatLargePageRecheck; /*< The number of times we rechecked a disabled large page./
3332	/** @} */
3333
3334	#ifdef VBOX_WITH_STATISTICS
3335	/** @name Statistics on the heap.
3336	* @{ */
3337	R3PTRTYPE(PGMSTATS *) pStatsR3;
3338	R0PTRTYPE(PGMSTATS *) pStatsR0;
3339	RCPTRTYPE(PGMSTATS *) pStatsRC;
3340	RTRCPTR RCPtrAlignment;
3341	/** @} */
3342	#endif
3343	} PGM;
3344	#ifndef IN_TSTVMSTRUCTGC /* HACK */
3345	AssertCompileMemberAlignment(PGM, paDynPageMap32BitPTEsGC, 8);
3346	AssertCompileMemberAlignment(PGM, GCPtrMappingFixed, sizeof(RTGCPTR));
3347	AssertCompileMemberAlignment(PGM, HCPhysInterPD, 8);
3348	AssertCompileMemberAlignment(PGM, CritSectX, 8);
3349	AssertCompileMemberAlignment(PGM, ChunkR3Map, 8);
3350	AssertCompileMemberAlignment(PGM, PhysTlbHC, 8);
3351	AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
3352	AssertCompileMemberAlignment(PGM, aHandyPages, 8);
3353	AssertCompileMemberAlignment(PGM, cRelocations, 8);
3354	#endif /* !IN_TSTVMSTRUCTGC */
3355	/** Pointer to the PGM instance data. */
3356	typedef PGM *PPGM;
3357
3358
3359
3360	typedef struct PGMCPUSTATS
3361	{
3362	/* Common */
3363	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
3364	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
3365
3366	/* R0 only: */
3367	STAMPROFILE StatR0NpMiscfg; /*< R0: PGMR0Trap0eHandlerNPMisconfig() profiling. /
3368	STAMCOUNTER StatR0NpMiscfgSyncPage; /*< R0: SyncPage calls from PGMR0Trap0eHandlerNPMisconfig(). /
3369
3370	/* RZ only: */
3371	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
3372	STAMPROFILE StatRZTrap0eTime2Ballooned; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is read access to a ballooned page. /
3373	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
3374	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
3375	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
3376	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
3377	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
3378	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
3379	STAMPROFILE StatRZTrap0eTime2InvalidPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access to an invalid physical guest address. /
3380	STAMPROFILE StatRZTrap0eTime2MakeWritable; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a page that needed to be made writable. /
3381	STAMPROFILE StatRZTrap0eTime2Mapping; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is the guest mappings. /
3382	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
3383	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
3384	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
3385	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
3386	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
3387	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
3388	STAMPROFILE StatRZTrap0eTime2WPEmulation; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CR0.WP emulation. /
3389	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
3390	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
3391	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
3392	STAMCOUNTER StatRZTrap0eHandlersPhysAll; /*< RC/R0: Number of traps due to physical all-access handlers. /
3393	STAMCOUNTER StatRZTrap0eHandlersPhysAllOpt; /*< RC/R0: Number of the physical all-access handler traps using the optimization. /
3394	STAMCOUNTER StatRZTrap0eHandlersPhysWrite; /*< RC/R0: Number of traps due to write-physical access handlers. /
3395	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
3396	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
3397	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
3398	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
3399	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
3400	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: \#PF err kind /
3401	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: \#PF err kind /
3402	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: \#PF err kind /
3403	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: \#PF err kind /
3404	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: \#PF err kind /
3405	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: \#PF err kind /
3406	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: \#PF err kind /
3407	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: \#PF err kind /
3408	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: \#PF err kind /
3409	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: \#PF err kind /
3410	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: \#PF err kind /
3411	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest \#PFs. /
3412	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest \#PF to HMA or other mapping. /
3413	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
3414	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
3415	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the \#PFs. /
3416	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
3417	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
3418	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
3419	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
3420	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
3421	STAMCOUNTER StatRZDynMapMigrateInvlPg; /*< RZ: invlpg in PGMR0DynMapMigrateAutoSet. /
3422	STAMPROFILE StatRZDynMapGCPageInl; /*< RZ: Calls to pgmRZDynMapGCPageInlined. /
3423	STAMCOUNTER StatRZDynMapGCPageInlHits; /*< RZ: Hash table lookup hits. /
3424	STAMCOUNTER StatRZDynMapGCPageInlMisses; /*< RZ: Misses that falls back to the code common. /
3425	STAMCOUNTER StatRZDynMapGCPageInlRamHits; /*< RZ: 1st ram range hits. /
3426	STAMCOUNTER StatRZDynMapGCPageInlRamMisses; /*< RZ: 1st ram range misses, takes slow path. /
3427	STAMPROFILE StatRZDynMapHCPageInl; /*< RZ: Calls to pgmRZDynMapHCPageInlined. /
3428	STAMCOUNTER StatRZDynMapHCPageInlHits; /*< RZ: Hash table lookup hits. /
3429	STAMCOUNTER StatRZDynMapHCPageInlMisses; /*< RZ: Misses that falls back to the code common. /
3430	STAMPROFILE StatRZDynMapHCPage; /*< RZ: Calls to pgmRZDynMapHCPageCommon. /
3431	STAMCOUNTER StatRZDynMapSetOptimize; /*< RZ: Calls to pgmRZDynMapOptimizeAutoSet. /
3432	STAMCOUNTER StatRZDynMapSetSearchFlushes; /*< RZ: Set search restoring to subset flushes. /
3433	STAMCOUNTER StatRZDynMapSetSearchHits; /*< RZ: Set search hits. /
3434	STAMCOUNTER StatRZDynMapSetSearchMisses; /*< RZ: Set search misses. /
3435	STAMCOUNTER StatRZDynMapPage; /*< RZ: Calls to pgmR0DynMapPage. /
3436	STAMCOUNTER StatRZDynMapPageHits0; /*< RZ: Hits at iPage+0. /
3437	STAMCOUNTER StatRZDynMapPageHits1; /*< RZ: Hits at iPage+1. /
3438	STAMCOUNTER StatRZDynMapPageHits2; /*< RZ: Hits at iPage+2. /
3439	STAMCOUNTER StatRZDynMapPageInvlPg; /*< RZ: invlpg. /
3440	STAMCOUNTER StatRZDynMapPageSlow; /*< RZ: Calls to pgmR0DynMapPageSlow. /
3441	STAMCOUNTER StatRZDynMapPageSlowLoopHits; /*< RZ: Hits in the pgmR0DynMapPageSlow search loop. /
3442	STAMCOUNTER StatRZDynMapPageSlowLoopMisses; /*< RZ: Misses in the pgmR0DynMapPageSlow search loop. /
3443	//STAMCOUNTER StatRZDynMapPageSlowLostHits; /*< RZ: Lost hits. /
3444	STAMCOUNTER StatRZDynMapSubsets; /*< RZ: Times PGMDynMapPushAutoSubset was called. /
3445	STAMCOUNTER StatRZDynMapPopFlushes; /*< RZ: Times PGMDynMapPopAutoSubset flushes the subset. /
3446	STAMCOUNTER aStatRZDynMapSetFilledPct[11]; /*< RZ: Set fill distribution, percent. /
3447
3448	/* HC - R3 and (maybe) R0: */
3449
3450	/* RZ & R3: */
3451	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
3452	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
3453	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
3454	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
3455	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
3456	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
3457	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
3458	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
3459	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
3460	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
3461	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
3462	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
3463	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
3464	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
3465	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
3466	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
3467	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
3468	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
3469	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
3470	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
3471	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
3472	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
3473	STAMCOUNTER StatRZDirtyPageStale; /*< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) /
3474	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
3475	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
3476	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
3477	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
3478	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
3479	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
3480	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
3481	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
3482	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
3483	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
3484	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
3485	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
3486	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. /
3487	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. /
3488	STAMCOUNTER StatRZPageOutOfSyncUserWrite; /*< RC/R0: The number of times user page is out of sync was detected in \#PF. /
3489	STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /*< RC/R0: The number of times supervisor page is out of sync was detected in in \#PF. /
3490	STAMCOUNTER StatRZPageOutOfSyncBallloon; /*< RC/R0: The number of times a ballooned page was accessed (read). /
3491	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
3492	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
3493	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
3494	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
3495	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
3496	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
3497	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
3498
3499	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
3500	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
3501	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
3502	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
3503	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
3504	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
3505	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
3506	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
3507	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
3508	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
3509	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
3510	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
3511	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
3512	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
3513	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
3514	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
3515	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
3516	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
3517	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
3518	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
3519	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
3520	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
3521	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
3522	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
3523	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
3524	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
3525	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
3526	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
3527	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
3528	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
3529	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
3530	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
3531	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
3532	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
3533	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in \#PF or VerifyAccessSyncPage. /
3534	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in \#PF or VerifyAccessSyncPage. /
3535	STAMCOUNTER StatR3PageOutOfSyncUserWrite; /*< R3: The number of times user page is out of sync was detected in \#PF. /
3536	STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /*< R3: The number of times supervisor page is out of sync was detected in in \#PF. /
3537	STAMCOUNTER StatR3PageOutOfSyncBallloon; /*< R3: The number of times a ballooned page was accessed (read). /
3538	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
3539	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
3540	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
3541	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
3542	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
3543	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
3544	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
3545	/** @} */
3546	} PGMCPUSTATS;
3547
3548
3549	/**
3550	* Converts a PGMCPU pointer into a VM pointer.
3551	* @returns Pointer to the VM structure the PGM is part of.
3552	* @param pPGM Pointer to PGMCPU instance data.
3553	*/
3554	#define PGMCPU2VM(pPGM) ( (PVM)((char*)(pPGM) - (pPGM)->offVM) )
3555
3556	/**
3557	* Converts a PGMCPU pointer into a PGM pointer.
3558	* @returns Pointer to the VM structure the PGM is part of.
3559	* @param pPGM Pointer to PGMCPU instance data.
3560	*/
3561	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char *)(pPGMCpu) - (pPGMCpu)->offPGM) )
3562
3563	/**
3564	* PGMCPU Data (part of VMCPU).
3565	*/
3566	typedef struct PGMCPU
3567	{
3568	/** Offset to the VM structure. */
3569	int32_t offVM;
3570	/** Offset to the VMCPU structure. */
3571	int32_t offVCpu;
3572	/** Offset of the PGM structure relative to VMCPU. */
3573	int32_t offPGM;
3574	uint32_t uPadding0; /*< structure size alignment. /
3575
3576	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE) \|\| defined(VBOX_WITH_RAW_MODE)
3577	/** Automatically tracked physical memory mapping set.
3578	* Ring-0 and strict raw-mode builds. */
3579	PGMMAPSET AutoSet;
3580	#endif
3581
3582	/** A20 gate mask.
3583	* Our current approach to A20 emulation is to let REM do it and don't bother
3584	* anywhere else. The interesting Guests will be operating with it enabled anyway.
3585	* But whould need arrise, we'll subject physical addresses to this mask. */
3586	RTGCPHYS GCPhysA20Mask;
3587	/** A20 gate state - boolean! */
3588	bool fA20Enabled;
3589	/** Mirror of the EFER.NXE bit. Managed by PGMNotifyNxeChanged. */
3590	bool fNoExecuteEnabled;
3591	/** Unused bits. */
3592	bool afUnused[2];
3593
3594	/** What needs syncing (PGM_SYNC_*).
3595	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
3596	* PGMFlushTLB, and PGMR3Load. */
3597	RTUINT fSyncFlags;
3598
3599	/** The shadow paging mode. */
3600	PGMMODE enmShadowMode;
3601	/** The guest paging mode. */
3602	PGMMODE enmGuestMode;
3603
3604	/** The current physical address representing in the guest CR3 register. */
3605	RTGCPHYS GCPhysCR3;
3606
3607	/** @name 32-bit Guest Paging.
3608	* @{ */
3609	/** The guest's page directory, R3 pointer. */
3610	R3PTRTYPE(PX86PD) pGst32BitPdR3;
3611	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3612	/** The guest's page directory, R0 pointer. */
3613	R0PTRTYPE(PX86PD) pGst32BitPdR0;
3614	#endif
3615	/** The guest's page directory, static RC mapping. */
3616	RCPTRTYPE(PX86PD) pGst32BitPdRC;
3617	/** Mask containing the MBZ bits of a big page PDE. */
3618	uint32_t fGst32BitMbzBigPdeMask;
3619	/** Set if the page size extension (PSE) is enabled. */
3620	bool fGst32BitPageSizeExtension;
3621	/** Alignment padding. */
3622	bool afAlignment2[3];
3623	/** @} */
3624
3625	/** @name PAE Guest Paging.
3626	* @{ */
3627	/** The guest's page directory pointer table, static RC mapping. */
3628	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
3629	/** The guest's page directory pointer table, R3 pointer. */
3630	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
3631	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3632	/** The guest's page directory pointer table, R0 pointer. */
3633	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
3634	#endif
3635
3636	/** The guest's page directories, R3 pointers.
3637	* These are individual pointers and don't have to be adjacent.
3638	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
3639	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
3640	/** The guest's page directories, R0 pointers.
3641	* Same restrictions as apGstPaePDsR3. */
3642	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3643	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
3644	#endif
3645	/** The guest's page directories, static GC mapping.
3646	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
3647	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
3648	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
3649	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC.
3650	* @todo Remove this and use aGstPaePdpeRegs instead? */
3651	RTGCPHYS aGCPhysGstPaePDs[4];
3652	/** The values of the 4 PDPE CPU registers (PAE). */
3653	X86PDPE aGstPaePdpeRegs[4];
3654	/** The physical addresses of the monitored guest page directories (PAE). */
3655	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
3656	/** Mask containing the MBZ PTE bits. */
3657	uint64_t fGstPaeMbzPteMask;
3658	/** Mask containing the MBZ PDE bits. */
3659	uint64_t fGstPaeMbzPdeMask;
3660	/** Mask containing the MBZ big page PDE bits. */
3661	uint64_t fGstPaeMbzBigPdeMask;
3662	/** Mask containing the MBZ PDPE bits. */
3663	uint64_t fGstPaeMbzPdpeMask;
3664	/** @} */
3665
3666	/** @name AMD64 Guest Paging.
3667	* @{ */
3668	/** The guest's page directory pointer table, R3 pointer. */
3669	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
3670	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
3671	/** The guest's page directory pointer table, R0 pointer. */
3672	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
3673	#else
3674	RTR0PTR alignment6b; /*< alignment equalizer. /
3675	#endif
3676	/** Mask containing the MBZ PTE bits. */
3677	uint64_t fGstAmd64MbzPteMask;
3678	/** Mask containing the MBZ PDE bits. */
3679	uint64_t fGstAmd64MbzPdeMask;
3680	/** Mask containing the MBZ big page PDE bits. */
3681	uint64_t fGstAmd64MbzBigPdeMask;
3682	/** Mask containing the MBZ PDPE bits. */
3683	uint64_t fGstAmd64MbzPdpeMask;
3684	/** Mask containing the MBZ big page PDPE bits. */
3685	uint64_t fGstAmd64MbzBigPdpeMask;
3686	/** Mask containing the MBZ PML4E bits. */
3687	uint64_t fGstAmd64MbzPml4eMask;
3688	/** Mask containing the PDPE bits that we shadow. */
3689	uint64_t fGstAmd64ShadowedPdpeMask;
3690	/** Mask containing the PML4E bits that we shadow. */
3691	uint64_t fGstAmd64ShadowedPml4eMask;
3692	/** @} */
3693
3694	/** @name PAE and AMD64 Guest Paging.
3695	* @{ */
3696	/** Mask containing the PTE bits that we shadow. */
3697	uint64_t fGst64ShadowedPteMask;
3698	/** Mask containing the PDE bits that we shadow. */
3699	uint64_t fGst64ShadowedPdeMask;
3700	/** Mask containing the big page PDE bits that we shadow in the PDE. */
3701	uint64_t fGst64ShadowedBigPdeMask;
3702	/** Mask containing the big page PDE bits that we shadow in the PTE. */
3703	uint64_t fGst64ShadowedBigPde4PteMask;
3704	/** @} */
3705
3706	/** Pointer to the page of the current active CR3 - R3 Ptr. */
3707	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
3708	/** Pointer to the page of the current active CR3 - R0 Ptr. */
3709	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
3710	/** Pointer to the page of the current active CR3 - RC Ptr. */
3711	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
3712	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
3713	uint32_t iShwUser;
3714	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
3715	uint32_t iShwUserTable;
3716	# if HC_ARCH_BITS == 64
3717	RTRCPTR alignment6; /*< structure size alignment. /
3718	# endif
3719	/** @} */
3720
3721	/** @name Function pointers for Shadow paging.
3722	* @{
3723	*/
3724	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3725	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
3726	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3727	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
3728
3729	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3730	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
3731
3732	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3733	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags));
3734
3735	/** @} */
3736
3737	/** @name Function pointers for Guest paging.
3738	* @{
3739	*/
3740	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3741	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
3742	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3743	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3744	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3745	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3746	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3747	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3748	#if HC_ARCH_BITS == 64
3749	RTRCPTR alignment3; /*< structure size alignment. /
3750	#endif
3751
3752	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3753	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3754	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3755	/** @} */
3756
3757	/** @name Function pointers for Both Shadow and Guest paging.
3758	* @{
3759	*/
3760	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3761	/* no pfnR3BthTrap0eHandler */
3762	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3763	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3764	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3765	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3766	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3767	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3768	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
3769
3770	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
3771	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3772	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3773	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3774	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3775	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3776	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3777	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
3778
3779	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken));
3780	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3781	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3782	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3783	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3784	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3785	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3786	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
3787	#if 0
3788	RTRCPTR alignment2; /*< structure size alignment. /
3789	#endif
3790	/** @} */
3791
3792	/** For saving stack space, the disassembler state is allocated here instead of
3793	* on the stack.
3794	* @note The DISCPUSTATE structure is not R3/R0/RZ clean! */
3795	union
3796	{
3797	/** The disassembler scratch space. */
3798	DISCPUSTATE DisState;
3799	/** Padding. */
3800	uint8_t abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
3801	};
3802
3803	/** Count the number of pgm pool access handler calls. */
3804	uint64_t cPoolAccessHandler;
3805
3806	/** @name Release Statistics
3807	* @{ */
3808	/** The number of times the guest has switched mode since last reset or statistics reset. */
3809	STAMCOUNTER cGuestModeChanges;
3810	/** @} */
3811
3812	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
3813	/** @name Statistics
3814	* @{ */
3815	/** RC: Pointer to the statistics. */
3816	RCPTRTYPE(PGMCPUSTATS *) pStatsRC;
3817	/** RC: Which statistic this \#PF should be attributed to. */
3818	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
3819	/** R0: Pointer to the statistics. */
3820	R0PTRTYPE(PGMCPUSTATS *) pStatsR0;
3821	/** R0: Which statistic this \#PF should be attributed to. */
3822	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
3823	/** R3: Pointer to the statistics. */
3824	R3PTRTYPE(PGMCPUSTATS *) pStatsR3;
3825	/** Alignment padding. */
3826	RTR3PTR pPaddingR3;
3827	/** @} */
3828	#endif /* VBOX_WITH_STATISTICS */
3829	} PGMCPU;
3830	/** Pointer to the per-cpu PGM data. */
3831	typedef PGMCPU *PPGMCPU;
3832
3833
3834	/** @name PGM::fSyncFlags Flags
3835	* @{
3836	*/
3837	/** Updates the virtual access handler state bit in PGMPAGE. */
3838	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
3839	/** Always sync CR3. */
3840	#define PGM_SYNC_ALWAYS RT_BIT(1)
3841	/** Check monitoring on next CR3 (re)load and invalidate page.
3842	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
3843	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
3844	/** Check guest mapping in SyncCR3. */
3845	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
3846	/** Clear the page pool (a light weight flush). */
3847	#define PGM_SYNC_CLEAR_PGM_POOL_BIT 8
3848	#define PGM_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
3849	/** @} */
3850
3851
3852	RT_C_DECLS_BEGIN
3853
3854	int pgmLock(PVM pVM);
3855	void pgmUnlock(PVM pVM);
3856	/**
3857	* Asserts that the caller owns the PDM lock.
3858	* This is the internal variant of PGMIsLockOwner.
3859	* @param a_pVM The VM handle.
3860	*/
3861	#define PGM_LOCK_ASSERT_OWNER(a_pVM) Assert(PDMCritSectIsOwner(&(a_pVM)->pgm.s.CritSectX))
3862	/**
3863	* Asserts that the caller owns the PDM lock.
3864	* This is the internal variant of PGMIsLockOwner.
3865	* @param a_pVM The VM handle.
3866	* @param a_pVCpu The current CPU handle.
3867	*/
3868	#define PGM_LOCK_ASSERT_OWNER_EX(a_pVM, a_pVCpu) Assert(PDMCritSectIsOwnerEx(&(a_pVM)->pgm.s.CritSectX, pVCpu))
3869
3870	int pgmR3MappingsFixInternal(PVM pVM, RTGCPTR GCPtrBase, uint32_t cb);
3871	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
3872	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
3873	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
3874	int pgmMapResolveConflicts(PVM pVM);
3875	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3876
3877	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
3878	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
3879	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, bool fDoAccounting);
3880	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
3881	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
3882	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
3883	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
3884	#else
3885	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
3886	#endif
3887	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3888	int pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
3889
3890	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3891	int pgmPhysAllocLargePage(PVM pVM, RTGCPHYS GCPhys);
3892	int pgmPhysRecheckLargePage(PVM pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage);
3893	int pgmPhysPageLoadIntoTlb(PVM pVM, RTGCPHYS GCPhys);
3894	int pgmPhysPageLoadIntoTlbWithPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3895	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage);
3896	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3897	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3898	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3899	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
3900	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
3901	int pgmPhysGCPhys2R3Ptr(PVM pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr);
3902	int pgmPhysGCPhys2CCPtrInternalDepr(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3903	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock);
3904	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock);
3905	void pgmPhysReleaseInternalPageMappingLock(PVM pVM, PPGMPAGEMAPLOCK pLock);
3906	VMMDECL(int) pgmPhysHandlerRedirectToHC(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
3907	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
3908	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys);
3909	void pgmPhysInvalidRamRangeTlbs(PVM pVM);
3910	void pgmPhysInvalidatePageMapTLB(PVM pVM);
3911	void pgmPhysInvalidatePageMapTLBEntry(PVM pVM, RTGCPHYS GCPhys);
3912	PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys);
3913	PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys);
3914	PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys);
3915	int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage);
3916	int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam);
3917
3918	#ifdef IN_RING3
3919	void pgmR3PhysRelinkRamRanges(PVM pVM);
3920	int pgmR3PhysRamPreAllocate(PVM pVM);
3921	int pgmR3PhysRamReset(PVM pVM);
3922	int pgmR3PhysRomReset(PVM pVM);
3923	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
3924	int pgmR3PhysRamTerm(PVM pVM);
3925	void pgmR3PhysRomTerm(PVM pVM);
3926
3927	int pgmR3PoolInit(PVM pVM);
3928	void pgmR3PoolRelocate(PVM pVM);
3929	void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu);
3930	void pgmR3PoolReset(PVM pVM);
3931	void pgmR3PoolClearAll(PVM pVM, bool fFlushRemTlb);
3932	DECLCALLBACK(VBOXSTRICTRC) pgmR3PoolClearAllRendezvous(PVM pVM, PVMCPU pVCpu, void *fpvFlushRemTbl);
3933	void pgmR3PoolWriteProtectPages(PVM pVM);
3934
3935	#endif /* IN_RING3 */
3936	#if defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0) \|\| defined(IN_RC)
3937	int pgmRZDynMapHCPageCommon(PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
3938	int pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL);
3939	# ifdef LOG_ENABLED
3940	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint, RT_SRC_POS_DECL);
3941	# else
3942	void pgmRZDynMapUnusedHint(PVMCPU pVCpu, void *pvHint);
3943	# endif
3944	#endif
3945	int pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser,
3946	uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage);
3947
3948	DECLINLINE(int) pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable,
3949	PPPGMPOOLPAGE ppPage)
3950	{
3951	return pgmPoolAllocEx(pVM, GCPhys, enmKind, PGMPOOLACCESS_DONTCARE, iUser, iUserTable, false, ppPage);
3952	}
3953
3954	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
3955	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
3956	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush = true /* DO NOT USE false UNLESS YOU KNOWN WHAT YOU'RE DOING!! */);
3957	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
3958	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
3959	PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys);
3960	int pgmPoolSyncCR3(PVMCPU pVCpu);
3961	bool pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys);
3962	void pgmPoolInvalidateDirtyPage(PVM pVM, RTGCPHYS GCPhysPT);
3963	int pgmPoolTrackUpdateGCPhys(PVM pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
3964	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte);
3965	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte);
3966	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage, uint16_t iPte);
3967	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, unsigned cbWrite);
3968	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3969	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3970
3971	void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3972	void pgmPoolResetDirtyPages(PVM pVM);
3973	void pgmPoolResetDirtyPage(PVM pVM, RTGCPTR GCPtrPage);
3974
3975	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
3976	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
3977
3978	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
3979	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
3980	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
3981	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
3982
3983	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
3984	int pgmShwSyncNestedPageLocked(PVMCPU pVCpu, RTGCPHYS GCPhysFault, uint32_t cPages, PGMMODE enmShwPagingMode);
3985
3986	int pgmGstLazyMap32BitPD(PVMCPU pVCpu, PX86PD *ppPd);
3987	int pgmGstLazyMapPaePDPT(PVMCPU pVCpu, PX86PDPT *ppPdpt);
3988	int pgmGstLazyMapPaePD(PVMCPU pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd);
3989	int pgmGstLazyMapPml4(PVMCPU pVCpu, PX86PML4 *ppPml4);
3990
3991	# if defined(VBOX_STRICT) && HC_ARCH_BITS == 64
3992	DECLCALLBACK(int) pgmR3CmdCheckDuplicatePages(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs);
3993	DECLCALLBACK(int) pgmR3CmdShowSharedModules(PCDBGCCMD pCmd, PDBGCCMDHLP pCmdHlp, PVM pVM, PCDBGCVAR paArgs, unsigned cArgs);
3994	# endif
3995
3996	RT_C_DECLS_END
3997
3998	/** @} */
3999
4000	#endif
4001

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/include/PGMInternal.h@ 38953

Download in other formats: