PGMInternal.h@ 19834

Last change on this file since 19834 was 19834, checked in by vboxsync, 16 years ago
Comment update
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 184.7 KB

Line
1	/* $Id: PGMInternal.h 19834 2009-05-19 15:16:08Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	#ifndef ___PGMInternal_h
23	#define ___PGMInternal_h
24
25	#include <VBox/cdefs.h>
26	#include <VBox/types.h>
27	#include <VBox/err.h>
28	#include <VBox/stam.h>
29	#include <VBox/param.h>
30	#include <VBox/vmm.h>
31	#include <VBox/mm.h>
32	#include <VBox/pdmcritsect.h>
33	#include <VBox/pdmapi.h>
34	#include <VBox/dis.h>
35	#include <VBox/dbgf.h>
36	#include <VBox/log.h>
37	#include <VBox/gmm.h>
38	#include <VBox/hwaccm.h>
39	#include <iprt/avl.h>
40	#include <iprt/assert.h>
41	#include <iprt/critsect.h>
42
43
44
45	/** @defgroup grp_pgm_int Internals
46	* @ingroup grp_pgm
47	* @internal
48	* @{
49	*/
50
51
52	/** @name PGM Compile Time Config
53	* @{
54	*/
55
56	/**
57	* Solve page is out of sync issues inside Guest Context (in PGMGC.cpp).
58	* Comment it if it will break something.
59	*/
60	#define PGM_OUT_OF_SYNC_IN_GC
61
62	/**
63	* Check and skip global PDEs for non-global flushes
64	*/
65	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
66
67	/**
68	* Sync N pages instead of a whole page table
69	*/
70	#define PGM_SYNC_N_PAGES
71
72	/**
73	* Number of pages to sync during a page fault
74	*
75	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
76	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
77	*/
78	#define PGM_SYNC_NR_PAGES 8
79
80	/**
81	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
82	*/
83	#define PGM_MAX_PHYSCACHE_ENTRIES 64
84	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
85
86	/**
87	* Enable caching of PGMR3PhysRead/WriteByte/Word/Dword
88	*/
89	#define PGM_PHYSMEMACCESS_CACHING
90
91	/** @def PGMPOOL_WITH_CACHE
92	* Enable agressive caching using the page pool.
93	*
94	* This requires PGMPOOL_WITH_USER_TRACKING and PGMPOOL_WITH_MONITORING.
95	*/
96	#define PGMPOOL_WITH_CACHE
97
98	/** @def PGMPOOL_WITH_MIXED_PT_CR3
99	* When defined, we'll deal with 'uncachable' pages.
100	*/
101	#ifdef PGMPOOL_WITH_CACHE
102	# define PGMPOOL_WITH_MIXED_PT_CR3
103	#endif
104
105	/** @def PGMPOOL_WITH_MONITORING
106	* Monitor the guest pages which are shadowed.
107	* When this is enabled, PGMPOOL_WITH_CACHE or PGMPOOL_WITH_GCPHYS_TRACKING must
108	* be enabled as well.
109	* @remark doesn't really work without caching now. (Mixed PT/CR3 change.)
110	*/
111	#ifdef PGMPOOL_WITH_CACHE
112	# define PGMPOOL_WITH_MONITORING
113	#endif
114
115	/** @def PGMPOOL_WITH_GCPHYS_TRACKING
116	* Tracking the of shadow pages mapping guest physical pages.
117	*
118	* This is very expensive, the current cache prototype is trying to figure out
119	* whether it will be acceptable with an agressive caching policy.
120	*/
121	#if defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
122	# define PGMPOOL_WITH_GCPHYS_TRACKING
123	#endif
124
125	/** @def PGMPOOL_WITH_USER_TRACKING
126	* Tracking users of shadow pages. This is required for the linking of shadow page
127	* tables and physical guest addresses.
128	*/
129	#if defined(PGMPOOL_WITH_GCPHYS_TRACKING) \|\| defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
130	# define PGMPOOL_WITH_USER_TRACKING
131	#endif
132
133	/** @def PGMPOOL_CFG_MAX_GROW
134	* The maximum number of pages to add to the pool in one go.
135	*/
136	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
137
138	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
139	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
140	*/
141	#ifdef VBOX_STRICT
142	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
143	#endif
144
145	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
146	* Enables the experimental lazy page allocation code. */
147	/# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
148
149	/** @} */
150
151
152	/** @name PDPT and PML4 flags.
153	* These are placed in the three bits available for system programs in
154	* the PDPT and PML4 entries.
155	* @{ */
156	/** The entry is a permanent one and it's must always be present.
157	* Never free such an entry. */
158	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
159	/** Mapping (hypervisor allocated pagetable). */
160	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
161	/** @} */
162
163	/** @name Page directory flags.
164	* These are placed in the three bits available for system programs in
165	* the page directory entries.
166	* @{ */
167	/** Mapping (hypervisor allocated pagetable). */
168	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
169	/** Made read-only to facilitate dirty bit tracking. */
170	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
171	/** @} */
172
173	/** @name Page flags.
174	* These are placed in the three bits available for system programs in
175	* the page entries.
176	* @{ */
177	/** Made read-only to facilitate dirty bit tracking. */
178	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
179
180	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
181	/** Scanned and approved by CSAM (tm).
182	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
183	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
184	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
185	#endif
186
187	/** @} */
188
189	/** @name Defines used to indicate the shadow and guest paging in the templates.
190	* @{ */
191	#define PGM_TYPE_REAL 1
192	#define PGM_TYPE_PROT 2
193	#define PGM_TYPE_32BIT 3
194	#define PGM_TYPE_PAE 4
195	#define PGM_TYPE_AMD64 5
196	#define PGM_TYPE_NESTED 6
197	#define PGM_TYPE_EPT 7
198	#define PGM_TYPE_MAX PGM_TYPE_EPT
199	/** @} */
200
201	/** Macro for checking if the guest is using paging.
202	* @param uGstType PGM_TYPE_*
203	* @param uShwType PGM_TYPE_*
204	* @remark ASSUMES certain order of the PGM_TYPE_* values.
205	*/
206	#define PGM_WITH_PAGING(uGstType, uShwType) \
207	( (uGstType) >= PGM_TYPE_32BIT \
208	&& (uShwType) != PGM_TYPE_NESTED \
209	&& (uShwType) != PGM_TYPE_EPT)
210
211	/** Macro for checking if the guest supports the NX bit.
212	* @param uGstType PGM_TYPE_*
213	* @param uShwType PGM_TYPE_*
214	* @remark ASSUMES certain order of the PGM_TYPE_* values.
215	*/
216	#define PGM_WITH_NX(uGstType, uShwType) \
217	( (uGstType) >= PGM_TYPE_PAE \
218	&& (uShwType) != PGM_TYPE_NESTED \
219	&& (uShwType) != PGM_TYPE_EPT)
220
221
222	/** @def PGM_HCPHYS_2_PTR
223	* Maps a HC physical page pool address to a virtual address.
224	*
225	* @returns VBox status code.
226	* @param pVM The VM handle.
227	* @param HCPhys The HC physical address to map to a virtual one.
228	* @param ppv Where to store the virtual address. No need to cast this.
229	*
230	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
231	* small page window employeed by that function. Be careful.
232	* @remark There is no need to assert on the result.
233	*/
234	#ifdef IN_RC
235	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
236	PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
237	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
238	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
239	pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
240	#else
241	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
242	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
243	#endif
244
245	/** @def PGM_HCPHYS_2_PTR_BY_PGM
246	* Maps a HC physical page pool address to a virtual address.
247	*
248	* @returns VBox status code.
249	* @param pPGM The PGM instance data.
250	* @param HCPhys The HC physical address to map to a virtual one.
251	* @param ppv Where to store the virtual address. No need to cast this.
252	*
253	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
254	* small page window employeed by that function. Be careful.
255	* @remark There is no need to assert on the result.
256	*/
257	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
258	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
259	pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
260	#else
261	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
262	PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
263	#endif
264
265	/** @def PGM_GCPHYS_2_PTR
266	* Maps a GC physical page address to a virtual address.
267	*
268	* @returns VBox status code.
269	* @param pVM The VM handle.
270	* @param GCPhys The GC physical address to map to a virtual one.
271	* @param ppv Where to store the virtual address. No need to cast this.
272	*
273	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
274	* small page window employeed by that function. Be careful.
275	* @remark There is no need to assert on the result.
276	*/
277	#ifdef IN_RC
278	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
279	PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
280	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
281	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
282	pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
283	#else
284	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
285	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
286	#endif
287
288	/** @def PGM_GCPHYS_2_PTR_BY_PGMCPU
289	* Maps a GC physical page address to a virtual address.
290	*
291	* @returns VBox status code.
292	* @param pPGM Pointer to the PGM instance data.
293	* @param GCPhys The GC physical address to map to a virtual one.
294	* @param ppv Where to store the virtual address. No need to cast this.
295	*
296	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
297	* small page window employeed by that function. Be careful.
298	* @remark There is no need to assert on the result.
299	*/
300	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
301	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
302	pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), GCPhys, (void **)(ppv))
303	#else
304	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
305	PGM_GCPHYS_2_PTR(PGMCPU2VM(pPGM), GCPhys, ppv)
306	#endif
307
308	/** @def PGM_GCPHYS_2_PTR_EX
309	* Maps a unaligned GC physical page address to a virtual address.
310	*
311	* @returns VBox status code.
312	* @param pVM The VM handle.
313	* @param GCPhys The GC physical address to map to a virtual one.
314	* @param ppv Where to store the virtual address. No need to cast this.
315	*
316	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
317	* small page window employeed by that function. Be careful.
318	* @remark There is no need to assert on the result.
319	*/
320	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
321	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
322	PGMDynMapGCPageOff(pVM, GCPhys, (void **)(ppv))
323	#else
324	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
325	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
326	#endif
327
328	/** @def PGM_INVL_PG
329	* Invalidates a page when in GC does nothing in HC.
330	*
331	* @param GCVirt The virtual address of the page to invalidate.
332	*/
333	#ifdef IN_RC
334	# define PGM_INVL_PG(pVCpu, GCVirt) ASMInvalidatePage((void *)(GCVirt))
335	#elif defined(IN_RING0)
336	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
337	#else
338	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
339	#endif
340
341	/** @def PGM_INVL_BIG_PG
342	* Invalidates a 4MB page directory entry when in GC does nothing in HC.
343	*
344	* @param GCVirt The virtual address within the page directory to invalidate.
345	*/
346	#ifdef IN_RC
347	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) ASMReloadCR3()
348	#elif defined(IN_RING0)
349	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
350	#else
351	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
352	#endif
353
354	/** @def PGM_INVL_VCPU_TLBS()
355	* Invalidates the TLBs of the specified VCPU
356	*/
357	#ifdef IN_RC
358	# define PGM_INVL_VCPU_TLBS(pVCpu) ASMReloadCR3()
359	#elif defined(IN_RING0)
360	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
361	#else
362	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
363	#endif
364
365	/** Size of the GCPtrConflict array in PGMMAPPING.
366	* @remarks Must be a power of two. */
367	#define PGMMAPPING_CONFLICT_MAX 8
368
369	/**
370	* Structure for tracking GC Mappings.
371	*
372	* This structure is used by linked list in both GC and HC.
373	*/
374	typedef struct PGMMAPPING
375	{
376	/** Pointer to next entry. */
377	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
378	/** Pointer to next entry. */
379	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
380	/** Pointer to next entry. */
381	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
382	/** Indicate whether this entry is finalized. */
383	bool fFinalized;
384	/** Start Virtual address. */
385	RTGCPTR GCPtr;
386	/** Last Virtual address (inclusive). */
387	RTGCPTR GCPtrLast;
388	/** Range size (bytes). */
389	RTGCPTR cb;
390	/** Pointer to relocation callback function. */
391	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
392	/** User argument to the callback. */
393	R3PTRTYPE(void *) pvUser;
394	/** Mapping description / name. For easing debugging. */
395	R3PTRTYPE(const char *) pszDesc;
396	/** Last 8 addresses that caused conflicts. */
397	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
398	/** Number of conflicts for this hypervisor mapping. */
399	uint32_t cConflicts;
400	/** Number of page tables. */
401	uint32_t cPTs;
402
403	/** Array of page table mapping data. Each entry
404	* describes one page table. The array can be longer
405	* than the declared length.
406	*/
407	struct
408	{
409	/** The HC physical address of the page table. */
410	RTHCPHYS HCPhysPT;
411	/** The HC physical address of the first PAE page table. */
412	RTHCPHYS HCPhysPaePT0;
413	/** The HC physical address of the second PAE page table. */
414	RTHCPHYS HCPhysPaePT1;
415	/** The HC virtual address of the 32-bit page table. */
416	R3PTRTYPE(PX86PT) pPTR3;
417	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
418	R3PTRTYPE(PX86PTPAE) paPaePTsR3;
419	/** The GC virtual address of the 32-bit page table. */
420	RCPTRTYPE(PX86PT) pPTRC;
421	/** The GC virtual address of the two PAE page table. */
422	RCPTRTYPE(PX86PTPAE) paPaePTsRC;
423	/** The GC virtual address of the 32-bit page table. */
424	R0PTRTYPE(PX86PT) pPTR0;
425	/** The GC virtual address of the two PAE page table. */
426	R0PTRTYPE(PX86PTPAE) paPaePTsR0;
427	} aPTs[1];
428	} PGMMAPPING;
429	/** Pointer to structure for tracking GC Mappings. */
430	typedef struct PGMMAPPING *PPGMMAPPING;
431
432
433	/**
434	* Physical page access handler structure.
435	*
436	* This is used to keep track of physical address ranges
437	* which are being monitored in some kind of way.
438	*/
439	typedef struct PGMPHYSHANDLER
440	{
441	AVLROGCPHYSNODECORE Core;
442	/** Access type. */
443	PGMPHYSHANDLERTYPE enmType;
444	/** Number of pages to update. */
445	uint32_t cPages;
446	/** Pointer to R3 callback function. */
447	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
448	/** User argument for R3 handlers. */
449	R3PTRTYPE(void *) pvUserR3;
450	/** Pointer to R0 callback function. */
451	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
452	/** User argument for R0 handlers. */
453	R0PTRTYPE(void *) pvUserR0;
454	/** Pointer to GC callback function. */
455	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
456	/** User argument for RC handlers. */
457	RCPTRTYPE(void *) pvUserRC;
458	/** Description / Name. For easing debugging. */
459	R3PTRTYPE(const char *) pszDesc;
460	#ifdef VBOX_WITH_STATISTICS
461	/** Profiling of this handler. */
462	STAMPROFILE Stat;
463	#endif
464	} PGMPHYSHANDLER;
465	/** Pointer to a physical page access handler structure. */
466	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
467
468
469	/**
470	* Cache node for the physical addresses covered by a virtual handler.
471	*/
472	typedef struct PGMPHYS2VIRTHANDLER
473	{
474	/** Core node for the tree based on physical ranges. */
475	AVLROGCPHYSNODECORE Core;
476	/** Offset from this struct to the PGMVIRTHANDLER structure. */
477	int32_t offVirtHandler;
478	/** Offset of the next alias relative to this one.
479	* Bit 0 is used for indicating whether we're in the tree.
480	* Bit 1 is used for indicating that we're the head node.
481	*/
482	int32_t offNextAlias;
483	} PGMPHYS2VIRTHANDLER;
484	/** Pointer to a phys to virtual handler structure. */
485	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
486
487	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
488	* node is in the tree. */
489	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
490	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
491	* node is in the head of an alias chain.
492	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
493	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
494	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
495	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
496
497
498	/**
499	* Virtual page access handler structure.
500	*
501	* This is used to keep track of virtual address ranges
502	* which are being monitored in some kind of way.
503	*/
504	typedef struct PGMVIRTHANDLER
505	{
506	/** Core node for the tree based on virtual ranges. */
507	AVLROGCPTRNODECORE Core;
508	/** Size of the range (in bytes). */
509	RTGCPTR cb;
510	/** Number of cache pages. */
511	uint32_t cPages;
512	/** Access type. */
513	PGMVIRTHANDLERTYPE enmType;
514	/** Pointer to the RC callback function. */
515	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
516	#if HC_ARCH_BITS == 64
517	RTRCPTR padding;
518	#endif
519	/** Pointer to the R3 callback function for invalidation. */
520	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
521	/** Pointer to the R3 callback function. */
522	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
523	/** Description / Name. For easing debugging. */
524	R3PTRTYPE(const char *) pszDesc;
525	#ifdef VBOX_WITH_STATISTICS
526	/** Profiling of this handler. */
527	STAMPROFILE Stat;
528	#endif
529	/** Array of cached physical addresses for the monitored ranged. */
530	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
531	} PGMVIRTHANDLER;
532	/** Pointer to a virtual page access handler structure. */
533	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
534
535
536	/**
537	* Page type.
538	*
539	* @remarks This enum has to fit in a 3-bit field (see PGMPAGE::u3Type).
540	* @remarks This is used in the saved state, so changes to it requires bumping
541	* the saved state version.
542	* @todo So, convert to \#defines!
543	*/
544	typedef enum PGMPAGETYPE
545	{
546	/** The usual invalid zero entry. */
547	PGMPAGETYPE_INVALID = 0,
548	/** RAM page. (RWX) */
549	PGMPAGETYPE_RAM,
550	/** MMIO2 page. (RWX) */
551	PGMPAGETYPE_MMIO2,
552	/** MMIO2 page aliased over an MMIO page. (RWX)
553	* See PGMHandlerPhysicalPageAlias(). */
554	PGMPAGETYPE_MMIO2_ALIAS_MMIO,
555	/** Shadowed ROM. (RWX) */
556	PGMPAGETYPE_ROM_SHADOW,
557	/** ROM page. (R-X) */
558	PGMPAGETYPE_ROM,
559	/** MMIO page. (---) */
560	PGMPAGETYPE_MMIO,
561	/** End of valid entries. */
562	PGMPAGETYPE_END
563	} PGMPAGETYPE;
564	AssertCompile(PGMPAGETYPE_END <= 7);
565
566	/** @name Page type predicates.
567	* @{ */
568	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
569	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
570	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
571	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
572	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
573	/** @} */
574
575
576	/**
577	* A Physical Guest Page tracking structure.
578	*
579	* The format of this structure is complicated because we have to fit a lot
580	* of information into as few bits as possible. The format is also subject
581	* to change (there is one comming up soon). Which means that for we'll be
582	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
583	* accessess to the structure.
584	*/
585	typedef struct PGMPAGE
586	{
587	/** The physical address and a whole lot of other stuff. All bits are used! */
588	RTHCPHYS HCPhysX;
589	/** The page state. */
590	uint32_t u2StateX : 2;
591	/** Flag indicating that a write monitored page was written to when set. */
592	uint32_t fWrittenToX : 1;
593	/** For later. */
594	uint32_t fSomethingElse : 1;
595	/** The Page ID.
596	* @todo Merge with HCPhysX once we've liberated HCPhysX of its stuff.
597	* The HCPhysX will then be 100% static. */
598	uint32_t idPageX : 28;
599	/** The page type (PGMPAGETYPE). */
600	uint32_t u3Type : 3;
601	/** The physical handler state (PGM_PAGE_HNDL_PHYS_STATE) /
602	uint32_t u2HandlerPhysStateX : 2;
603	/** The virtual handler state (PGM_PAGE_HNDL_VIRT_STATE) /
604	uint32_t u2HandlerVirtStateX : 2;
605	uint32_t u29B : 25;
606	} PGMPAGE;
607	AssertCompileSize(PGMPAGE, 16);
608	/** Pointer to a physical guest page. */
609	typedef PGMPAGE *PPGMPAGE;
610	/** Pointer to a const physical guest page. */
611	typedef const PGMPAGE *PCPGMPAGE;
612	/** Pointer to a physical guest page pointer. */
613	typedef PPGMPAGE *PPPGMPAGE;
614
615
616	/**
617	* Clears the page structure.
618	* @param pPage Pointer to the physical guest page tracking structure.
619	*/
620	#define PGM_PAGE_CLEAR(pPage) \
621	do { \
622	(pPage)->HCPhysX = 0; \
623	(pPage)->u2StateX = 0; \
624	(pPage)->fWrittenToX = 0; \
625	(pPage)->fSomethingElse = 0; \
626	(pPage)->idPageX = 0; \
627	(pPage)->u3Type = 0; \
628	(pPage)->u29B = 0; \
629	} while (0)
630
631	/**
632	* Initializes the page structure.
633	* @param pPage Pointer to the physical guest page tracking structure.
634	*/
635	#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
636	do { \
637	(pPage)->HCPhysX = (_HCPhys); \
638	(pPage)->u2StateX = (_uState); \
639	(pPage)->fWrittenToX = 0; \
640	(pPage)->fSomethingElse = 0; \
641	(pPage)->idPageX = (_idPage); \
642	/(pPage)->u3Type = (_uType); - later / \
643	PGM_PAGE_SET_TYPE(pPage, _uType); \
644	(pPage)->u29B = 0; \
645	} while (0)
646
647	/**
648	* Initializes the page structure of a ZERO page.
649	* @param pPage Pointer to the physical guest page tracking structure.
650	*/
651	#define PGM_PAGE_INIT_ZERO(pPage, pVM, _uType) \
652	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
653	/** Temporary hack. Replaced by PGM_PAGE_INIT_ZERO once the old code is kicked out. */
654	# define PGM_PAGE_INIT_ZERO_REAL(pPage, pVM, _uType) \
655	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
656
657
658	/** @name The Page state, PGMPAGE::u2StateX.
659	* @{ */
660	/** The zero page.
661	* This is a per-VM page that's never ever mapped writable. */
662	#define PGM_PAGE_STATE_ZERO 0
663	/** A allocated page.
664	* This is a per-VM page allocated from the page pool (or wherever
665	* we get MMIO2 pages from if the type is MMIO2).
666	*/
667	#define PGM_PAGE_STATE_ALLOCATED 1
668	/** A allocated page that's being monitored for writes.
669	* The shadow page table mappings are read-only. When a write occurs, the
670	* fWrittenTo member is set, the page remapped as read-write and the state
671	* moved back to allocated. */
672	#define PGM_PAGE_STATE_WRITE_MONITORED 2
673	/** The page is shared, aka. copy-on-write.
674	* This is a page that's shared with other VMs. */
675	#define PGM_PAGE_STATE_SHARED 3
676	/** @} */
677
678
679	/**
680	* Gets the page state.
681	* @returns page state (PGM_PAGE_STATE_*).
682	* @param pPage Pointer to the physical guest page tracking structure.
683	*/
684	#define PGM_PAGE_GET_STATE(pPage) ( (pPage)->u2StateX )
685
686	/**
687	* Sets the page state.
688	* @param pPage Pointer to the physical guest page tracking structure.
689	* @param _uState The new page state.
690	*/
691	#define PGM_PAGE_SET_STATE(pPage, _uState) \
692	do { (pPage)->u2StateX = (_uState); } while (0)
693
694
695	/**
696	* Gets the host physical address of the guest page.
697	* @returns host physical address (RTHCPHYS).
698	* @param pPage Pointer to the physical guest page tracking structure.
699	*/
700	#define PGM_PAGE_GET_HCPHYS(pPage) ( (pPage)->HCPhysX & UINT64_C(0x0000fffffffff000) )
701
702	/**
703	* Sets the host physical address of the guest page.
704	* @param pPage Pointer to the physical guest page tracking structure.
705	* @param _HCPhys The new host physical address.
706	*/
707	#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
708	do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0xffff000000000fff)) \
709	\| ((_HCPhys) & UINT64_C(0x0000fffffffff000)); } while (0)
710
711	/**
712	* Get the Page ID.
713	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
714	* @param pPage Pointer to the physical guest page tracking structure.
715	*/
716	#define PGM_PAGE_GET_PAGEID(pPage) ( (pPage)->idPageX )
717	/* later:
718	#define PGM_PAGE_GET_PAGEID(pPage) ( ((uint32_t)(pPage)->HCPhysX >> (48 - 12))
719	\| ((uint32_t)(pPage)->HCPhysX & 0xfff) )
720	*/
721	/**
722	* Sets the Page ID.
723	* @param pPage Pointer to the physical guest page tracking structure.
724	*/
725	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->idPageX = (_idPage); } while (0)
726	/* later:
727	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0x0000fffffffff000)) \
728	\| ((_idPage) & 0xfff) \
729	\| (((_idPage) & 0x0ffff000) << (48-12)); } while (0)
730	*/
731
732	/**
733	* Get the Chunk ID.
734	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
735	* @param pPage Pointer to the physical guest page tracking structure.
736	*/
737	#define PGM_PAGE_GET_CHUNKID(pPage) ( (pPage)->idPageX >> GMM_CHUNKID_SHIFT )
738	/* later:
739	#if GMM_CHUNKID_SHIFT == 12
740	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> 48) )
741	#elif GMM_CHUNKID_SHIFT > 12
742	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> (48 + (GMM_CHUNKID_SHIFT - 12)) )
743	#elif GMM_CHUNKID_SHIFT < 12
744	# define PGM_PAGE_GET_CHUNKID(pPage) ( ( (uint32_t)((pPage)->HCPhysX >> 48) << (12 - GMM_CHUNKID_SHIFT) ) \
745	\| ( (uint32_t)((pPage)->HCPhysX & 0xfff) >> GMM_CHUNKID_SHIFT ) )
746	#else
747	# error "GMM_CHUNKID_SHIFT isn't defined or something."
748	#endif
749	*/
750
751	/**
752	* Get the index of the page within the allocaiton chunk.
753	* @returns The page index.
754	* @param pPage Pointer to the physical guest page tracking structure.
755	*/
756	#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (pPage)->idPageX & GMM_PAGEID_IDX_MASK )
757	/* later:
758	#if GMM_CHUNKID_SHIFT <= 12
759	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & GMM_PAGEID_IDX_MASK) )
760	#else
761	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & 0xfff) \
762	\| ( (uint32_t)((pPage)->HCPhysX >> 48) & (RT_BIT_32(GMM_CHUNKID_SHIFT - 12) - 1) ) )
763	#endif
764	*/
765
766
767	/**
768	* Gets the page type.
769	* @returns The page type.
770	* @param pPage Pointer to the physical guest page tracking structure.
771	*/
772	#define PGM_PAGE_GET_TYPE(pPage) (pPage)->u3Type
773
774	/**
775	* Sets the page type.
776	* @param pPage Pointer to the physical guest page tracking structure.
777	* @param _enmType The new page type (PGMPAGETYPE).
778	*/
779	#define PGM_PAGE_SET_TYPE(pPage, _enmType) \
780	do { (pPage)->u3Type = (_enmType); } while (0)
781
782	/**
783	* Checks if the page is marked for MMIO.
784	* @returns true/false.
785	* @param pPage Pointer to the physical guest page tracking structure.
786	*/
787	#define PGM_PAGE_IS_MMIO(pPage) ( (pPage)->u3Type == PGMPAGETYPE_MMIO )
788
789	/**
790	* Checks if the page is backed by the ZERO page.
791	* @returns true/false.
792	* @param pPage Pointer to the physical guest page tracking structure.
793	*/
794	#define PGM_PAGE_IS_ZERO(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_ZERO )
795
796	/**
797	* Checks if the page is backed by a SHARED page.
798	* @returns true/false.
799	* @param pPage Pointer to the physical guest page tracking structure.
800	*/
801	#define PGM_PAGE_IS_SHARED(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_SHARED )
802
803
804	/**
805	* Marks the paget as written to (for GMM change monitoring).
806	* @param pPage Pointer to the physical guest page tracking structure.
807	*/
808	#define PGM_PAGE_SET_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 1; } while (0)
809
810	/**
811	* Clears the written-to indicator.
812	* @param pPage Pointer to the physical guest page tracking structure.
813	*/
814	#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 0; } while (0)
815
816	/**
817	* Checks if the page was marked as written-to.
818	* @returns true/false.
819	* @param pPage Pointer to the physical guest page tracking structure.
820	*/
821	#define PGM_PAGE_IS_WRITTEN_TO(pPage) ( (pPage)->fWrittenToX )
822
823
824	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateX).
825	*
826	* @remarks The values are assigned in order of priority, so we can calculate
827	* the correct state for a page with different handlers installed.
828	* @{ */
829	/** No handler installed. */
830	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
831	/** Monitoring is temporarily disabled. */
832	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
833	/** Write access is monitored. */
834	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
835	/** All access is monitored. */
836	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
837	/** @} */
838
839	/**
840	* Gets the physical access handler state of a page.
841	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
842	* @param pPage Pointer to the physical guest page tracking structure.
843	*/
844	#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) ( (pPage)->u2HandlerPhysStateX )
845
846	/**
847	* Sets the physical access handler state of a page.
848	* @param pPage Pointer to the physical guest page tracking structure.
849	* @param _uState The new state value.
850	*/
851	#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
852	do { (pPage)->u2HandlerPhysStateX = (_uState); } while (0)
853
854	/**
855	* Checks if the page has any physical access handlers, including temporariliy disabled ones.
856	* @returns true/false
857	* @param pPage Pointer to the physical guest page tracking structure.
858	*/
859	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE )
860
861	/**
862	* Checks if the page has any active physical access handlers.
863	* @returns true/false
864	* @param pPage Pointer to the physical guest page tracking structure.
865	*/
866	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
867
868
869	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateX).
870	*
871	* @remarks The values are assigned in order of priority, so we can calculate
872	* the correct state for a page with different handlers installed.
873	* @{ */
874	/** No handler installed. */
875	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
876	/* 1 is reserved so the lineup is identical with the physical ones. */
877	/** Write access is monitored. */
878	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
879	/** All access is monitored. */
880	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
881	/** @} */
882
883	/**
884	* Gets the virtual access handler state of a page.
885	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
886	* @param pPage Pointer to the physical guest page tracking structure.
887	*/
888	#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ( (pPage)->u2HandlerVirtStateX )
889
890	/**
891	* Sets the virtual access handler state of a page.
892	* @param pPage Pointer to the physical guest page tracking structure.
893	* @param _uState The new state value.
894	*/
895	#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
896	do { (pPage)->u2HandlerVirtStateX = (_uState); } while (0)
897
898	/**
899	* Checks if the page has any virtual access handlers.
900	* @returns true/false
901	* @param pPage Pointer to the physical guest page tracking structure.
902	*/
903	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ( (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
904
905	/**
906	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
907	* virtual handlers.
908	* @returns true/false
909	* @param pPage Pointer to the physical guest page tracking structure.
910	*/
911	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
912
913
914
915	/**
916	* Checks if the page has any access handlers, including temporarily disabled ones.
917	* @returns true/false
918	* @param pPage Pointer to the physical guest page tracking structure.
919	*/
920	#define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
921	( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE \
922	\|\| (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
923
924	/**
925	* Checks if the page has any active access handlers.
926	* @returns true/false
927	* @param pPage Pointer to the physical guest page tracking structure.
928	*/
929	#define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
930	( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
931	\|\| (pPage)->u2HandlerVirtStateX >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
932
933	/**
934	* Checks if the page has any active access handlers catching all accesses.
935	* @returns true/false
936	* @param pPage Pointer to the physical guest page tracking structure.
937	*/
938	#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
939	( (pPage)->u2HandlerPhysStateX == PGM_PAGE_HNDL_PHYS_STATE_ALL \
940	\|\| (pPage)->u2HandlerVirtStateX == PGM_PAGE_HNDL_VIRT_STATE_ALL )
941
942
943
944
945	/** @def PGM_PAGE_GET_TRACKING
946	* Gets the packed shadow page pool tracking data associated with a guest page.
947	* @returns uint16_t containing the data.
948	* @param pPage Pointer to the physical guest page tracking structure.
949	*/
950	#define PGM_PAGE_GET_TRACKING(pPage) \
951	( ((uint16_t )&(pPage)->HCPhysX + 3) )
952
953	/** @def PGM_PAGE_SET_TRACKING
954	* Sets the packed shadow page pool tracking data associated with a guest page.
955	* @param pPage Pointer to the physical guest page tracking structure.
956	* @param u16TrackingData The tracking data to store.
957	*/
958	#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
959	do { ((uint16_t )&(pPage)->HCPhysX + 3) = (u16TrackingData); } while (0)
960
961	/** @def PGM_PAGE_GET_TD_CREFS
962	* Gets the @a cRefs tracking data member.
963	* @returns cRefs.
964	* @param pPage Pointer to the physical guest page tracking structure.
965	*/
966	#define PGM_PAGE_GET_TD_CREFS(pPage) \
967	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
968
969	#define PGM_PAGE_GET_TD_IDX(pPage) \
970	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
971
972	/**
973	* Ram range for GC Phys to HC Phys conversion.
974	*
975	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
976	* conversions too, but we'll let MM handle that for now.
977	*
978	* This structure is used by linked lists in both GC and HC.
979	*/
980	typedef struct PGMRAMRANGE
981	{
982	/** Start of the range. Page aligned. */
983	RTGCPHYS GCPhys;
984	/** Size of the range. (Page aligned of course). */
985	RTGCPHYS cb;
986	/** Pointer to the next RAM range - for R3. */
987	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
988	/** Pointer to the next RAM range - for R0. */
989	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
990	/** Pointer to the next RAM range - for RC. */
991	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
992	/** PGM_RAM_RANGE_FLAGS_* flags. */
993	uint32_t fFlags;
994	/** Last address in the range (inclusive). Page aligned (-1). */
995	RTGCPHYS GCPhysLast;
996	/** Start of the HC mapping of the range. This is only used for MMIO2. */
997	R3PTRTYPE(void *) pvR3;
998	/** The range description. */
999	R3PTRTYPE(const char *) pszDesc;
1000	/** Pointer to self - R0 pointer. */
1001	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1002	/** Pointer to self - RC pointer. */
1003	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1004	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1005	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 1];
1006	/** Array of physical guest page tracking structures. */
1007	PGMPAGE aPages[1];
1008	} PGMRAMRANGE;
1009	/** Pointer to Ram range for GC Phys to HC Phys conversion. */
1010	typedef PGMRAMRANGE *PPGMRAMRANGE;
1011
1012	/** @name PGMRAMRANGE::fFlags
1013	* @{ */
1014	/** The RAM range is floating around as an independent guest mapping. */
1015	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1016	/** @} */
1017
1018
1019	/**
1020	* Per page tracking structure for ROM image.
1021	*
1022	* A ROM image may have a shadow page, in which case we may have
1023	* two pages backing it. This structure contains the PGMPAGE for
1024	* both while PGMRAMRANGE have a copy of the active one. It is
1025	* important that these aren't out of sync in any regard other
1026	* than page pool tracking data.
1027	*/
1028	typedef struct PGMROMPAGE
1029	{
1030	/** The page structure for the virgin ROM page. */
1031	PGMPAGE Virgin;
1032	/** The page structure for the shadow RAM page. */
1033	PGMPAGE Shadow;
1034	/** The current protection setting. */
1035	PGMROMPROT enmProt;
1036	/** Pad the structure size to a multiple of 8. */
1037	uint32_t u32Padding;
1038	} PGMROMPAGE;
1039	/** Pointer to a ROM page tracking structure. */
1040	typedef PGMROMPAGE *PPGMROMPAGE;
1041
1042
1043	/**
1044	* A registered ROM image.
1045	*
1046	* This is needed to keep track of ROM image since they generally
1047	* intrude into a PGMRAMRANGE. It also keeps track of additional
1048	* info like the two page sets (read-only virgin and read-write shadow),
1049	* the current state of each page.
1050	*
1051	* Because access handlers cannot easily be executed in a different
1052	* context, the ROM ranges needs to be accessible and in all contexts.
1053	*/
1054	typedef struct PGMROMRANGE
1055	{
1056	/** Pointer to the next range - R3. */
1057	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1058	/** Pointer to the next range - R0. */
1059	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1060	/** Pointer to the next range - RC. */
1061	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1062	/** Pointer alignment */
1063	RTRCPTR GCPtrAlignment;
1064	/** Address of the range. */
1065	RTGCPHYS GCPhys;
1066	/** Address of the last byte in the range. */
1067	RTGCPHYS GCPhysLast;
1068	/** Size of the range. */
1069	RTGCPHYS cb;
1070	/** The flags (PGMPHYS_ROM_FLAG_). /
1071	uint32_t fFlags;
1072	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1073	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 7 : 3];
1074	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1075	* This is used for strictness checks. */
1076	R3PTRTYPE(const void *) pvOriginal;
1077	/** The ROM description. */
1078	R3PTRTYPE(const char *) pszDesc;
1079	/** The per page tracking structures. */
1080	PGMROMPAGE aPages[1];
1081	} PGMROMRANGE;
1082	/** Pointer to a ROM range. */
1083	typedef PGMROMRANGE *PPGMROMRANGE;
1084
1085
1086	/**
1087	* A registered MMIO2 (= Device RAM) range.
1088	*
1089	* There are a few reason why we need to keep track of these
1090	* registrations. One of them is the deregistration & cleanup
1091	* stuff, while another is that the PGMRAMRANGE associated with
1092	* such a region may have to be removed from the ram range list.
1093	*
1094	* Overlapping with a RAM range has to be 100% or none at all. The
1095	* pages in the existing RAM range must not be ROM nor MMIO. A guru
1096	* meditation will be raised if a partial overlap or an overlap of
1097	* ROM pages is encountered. On an overlap we will free all the
1098	* existing RAM pages and put in the ram range pages instead.
1099	*/
1100	typedef struct PGMMMIO2RANGE
1101	{
1102	/** The owner of the range. (a device) */
1103	PPDMDEVINSR3 pDevInsR3;
1104	/** Pointer to the ring-3 mapping of the allocation. */
1105	RTR3PTR pvR3;
1106	/** Pointer to the next range - R3. */
1107	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1108	/** Whether it's mapped or not. */
1109	bool fMapped;
1110	/** Whether it's overlapping or not. */
1111	bool fOverlapping;
1112	/** The PCI region number.
1113	* @remarks This ASSUMES that nobody will ever really need to have multiple
1114	* PCI devices with matching MMIO region numbers on a single device. */
1115	uint8_t iRegion;
1116	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundrary. */
1117	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 1 : 5];
1118	/** The associated RAM range. */
1119	PGMRAMRANGE RamRange;
1120	} PGMMMIO2RANGE;
1121	/** Pointer to a MMIO2 range. */
1122	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1123
1124
1125
1126
1127	/**
1128	* PGMPhysRead/Write cache entry
1129	*/
1130	typedef struct PGMPHYSCACHEENTRY
1131	{
1132	/** R3 pointer to physical page. */
1133	R3PTRTYPE(uint8_t *) pbR3;
1134	/** GC Physical address for cache entry */
1135	RTGCPHYS GCPhys;
1136	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1137	RTGCPHYS u32Padding0; /*< alignment padding. /
1138	#endif
1139	} PGMPHYSCACHEENTRY;
1140
1141	/**
1142	* PGMPhysRead/Write cache to reduce REM memory access overhead
1143	*/
1144	typedef struct PGMPHYSCACHE
1145	{
1146	/** Bitmap of valid cache entries */
1147	uint64_t aEntries;
1148	/** Cache entries */
1149	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1150	} PGMPHYSCACHE;
1151
1152
1153	/** Pointer to an allocation chunk ring-3 mapping. */
1154	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1155	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1156	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1157
1158	/**
1159	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1160	*
1161	* The primary tree (Core) uses the chunk id as key.
1162	* The secondary tree (AgeCore) is used for ageing and uses ageing sequence number as key.
1163	*/
1164	typedef struct PGMCHUNKR3MAP
1165	{
1166	/** The key is the chunk id. */
1167	AVLU32NODECORE Core;
1168	/** The key is the ageing sequence number. */
1169	AVLLU32NODECORE AgeCore;
1170	/** The current age thingy. */
1171	uint32_t iAge;
1172	/** The current reference count. */
1173	uint32_t volatile cRefs;
1174	/** The current permanent reference count. */
1175	uint32_t volatile cPermRefs;
1176	/** The mapping address. */
1177	void *pv;
1178	} PGMCHUNKR3MAP;
1179
1180	/**
1181	* Allocation chunk ring-3 mapping TLB entry.
1182	*/
1183	typedef struct PGMCHUNKR3MAPTLBE
1184	{
1185	/** The chunk id. */
1186	uint32_t volatile idChunk;
1187	#if HC_ARCH_BITS == 64
1188	uint32_t u32Padding; /*< alignment padding. /
1189	#endif
1190	/** The chunk map. */
1191	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1192	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1193	#else
1194	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1195	#endif
1196	} PGMCHUNKR3MAPTLBE;
1197	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1198	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1199
1200	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1201	* @remark Must be a power of two value. */
1202	#define PGM_CHUNKR3MAPTLB_ENTRIES 32
1203
1204	/**
1205	* Allocation chunk ring-3 mapping TLB.
1206	*
1207	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1208	* At first glance this might look kinda odd since AVL trees are
1209	* supposed to give the most optimial lookup times of all trees
1210	* due to their balancing. However, take a tree with 1023 nodes
1211	* in it, that's 10 levels, meaning that most searches has to go
1212	* down 9 levels before they find what they want. This isn't fast
1213	* compared to a TLB hit. There is the factor of cache misses,
1214	* and of course the problem with trees and branch prediction.
1215	* This is why we use TLBs in front of most of the trees.
1216	*
1217	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1218	* difficult when we switch to the new inlined AVL trees (from kStuff).
1219	*/
1220	typedef struct PGMCHUNKR3MAPTLB
1221	{
1222	/** The TLB entries. */
1223	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1224	} PGMCHUNKR3MAPTLB;
1225
1226	/**
1227	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1228	* @returns Chunk TLB index.
1229	* @param idChunk The Chunk ID.
1230	*/
1231	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1232
1233
1234	/**
1235	* Ring-3 guest page mapping TLB entry.
1236	* @remarks used in ring-0 as well at the moment.
1237	*/
1238	typedef struct PGMPAGER3MAPTLBE
1239	{
1240	/** Address of the page. */
1241	RTGCPHYS volatile GCPhys;
1242	/** The guest page. */
1243	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1244	R3PTRTYPE(PPGMPAGE) volatile pPage;
1245	#else
1246	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1247	#endif
1248	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1249	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1250	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1251	#else
1252	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1253	#endif
1254	/** The address */
1255	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1256	R3PTRTYPE(void *) volatile pv;
1257	#else
1258	R3R0PTRTYPE(void *) volatile pv;
1259	#endif
1260	#if HC_ARCH_BITS == 32
1261	uint32_t u32Padding; /*< alignment padding. /
1262	#endif
1263	} PGMPAGER3MAPTLBE;
1264	/** Pointer to an entry in the HC physical TLB. */
1265	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1266
1267
1268	/** The number of entries in the ring-3 guest page mapping TLB.
1269	* @remarks The value must be a power of two. */
1270	#define PGM_PAGER3MAPTLB_ENTRIES 64
1271
1272	/**
1273	* Ring-3 guest page mapping TLB.
1274	* @remarks used in ring-0 as well at the moment.
1275	*/
1276	typedef struct PGMPAGER3MAPTLB
1277	{
1278	/** The TLB entries. */
1279	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1280	} PGMPAGER3MAPTLB;
1281	/** Pointer to the ring-3 guest page mapping TLB. */
1282	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1283
1284	/**
1285	* Calculates the index of the TLB entry for the specified guest page.
1286	* @returns Physical TLB index.
1287	* @param GCPhys The guest physical address.
1288	*/
1289	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1290
1291
1292	/**
1293	* Mapping cache usage set entry.
1294	*
1295	* @remarks 16-bit ints was choosen as the set is not expected to be used beyond
1296	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1297	* cache. If it's extended to include ring-3, well, then something will
1298	* have be changed here...
1299	*/
1300	typedef struct PGMMAPSETENTRY
1301	{
1302	/** The mapping cache index. */
1303	uint16_t iPage;
1304	/** The number of references.
1305	* The max is UINT16_MAX - 1. */
1306	uint16_t cRefs;
1307	#if HC_ARCH_BITS == 64
1308	uint32_t alignment;
1309	#endif
1310	/** Pointer to the page. */
1311	RTR0PTR pvPage;
1312	/** The physical address for this entry. */
1313	RTHCPHYS HCPhys;
1314	} PGMMAPSETENTRY;
1315	/** Pointer to a mapping cache usage set entry. */
1316	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1317
1318	/**
1319	* Mapping cache usage set.
1320	*
1321	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1322	* done during exits / traps. The set is
1323	*/
1324	typedef struct PGMMAPSET
1325	{
1326	/** The number of occupied entries.
1327	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1328	* dynamic mappings. */
1329	uint32_t cEntries;
1330	/** The start of the current subset.
1331	* This is UINT32_MAX if no subset is currently open. */
1332	uint32_t iSubset;
1333	/** The index of the current CPU, only valid if the set is open. */
1334	int32_t iCpu;
1335	#if HC_ARCH_BITS == 64
1336	uint32_t alignment;
1337	#endif
1338	/** The entries. */
1339	PGMMAPSETENTRY aEntries[64];
1340	/** HCPhys -> iEntry fast lookup table.
1341	* Use PGMMAPSET_HASH for hashing.
1342	* The entries may or may not be valid, check against cEntries. */
1343	uint8_t aiHashTable[128];
1344	} PGMMAPSET;
1345	/** Pointer to the mapping cache set. */
1346	typedef PGMMAPSET *PPGMMAPSET;
1347
1348	/** PGMMAPSET::cEntries value for a closed set. */
1349	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1350
1351	/** Hash function for aiHashTable. */
1352	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1353
1354	/** The max fill size (strict builds). */
1355	#define PGMMAPSET_MAX_FILL (64U * 80U / 100U)
1356
1357
1358	/** @name Context neutrual page mapper TLB.
1359	*
1360	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1361	* code is writting in a kind of context neutrual way. Time will show whether
1362	* this actually makes sense or not...
1363	*
1364	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1365	* context ends up using a global mapping cache on some platforms
1366	* (darwin).
1367	*
1368	* @{ */
1369	/** @typedef PPGMPAGEMAPTLB
1370	* The page mapper TLB pointer type for the current context. */
1371	/** @typedef PPGMPAGEMAPTLB
1372	* The page mapper TLB entry pointer type for the current context. */
1373	/** @typedef PPGMPAGEMAPTLB
1374	* The page mapper TLB entry pointer pointer type for the current context. */
1375	/** @def PGM_PAGEMAPTLB_ENTRIES
1376	* The number of TLB entries in the page mapper TLB for the current context. */
1377	/** @def PGM_PAGEMAPTLB_IDX
1378	* Calculate the TLB index for a guest physical address.
1379	* @returns The TLB index.
1380	* @param GCPhys The guest physical address. */
1381	/** @typedef PPGMPAGEMAP
1382	* Pointer to a page mapper unit for current context. */
1383	/** @typedef PPPGMPAGEMAP
1384	* Pointer to a page mapper unit pointer for current context. */
1385	#ifdef IN_RC
1386	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1387	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1388	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1389	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1390	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1391	typedef void * PPGMPAGEMAP;
1392	typedef void ** PPPGMPAGEMAP;
1393	//#elif IN_RING0
1394	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1395	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1396	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1397	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1398	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1399	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1400	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1401	#else
1402	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1403	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1404	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1405	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1406	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1407	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1408	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1409	#endif
1410	/** @} */
1411
1412
1413	/** @name PGM Pool Indexes.
1414	* Aka. the unique shadow page identifier.
1415	* @{ */
1416	/** NIL page pool IDX. */
1417	#define NIL_PGMPOOL_IDX 0
1418	/** The first normal index. */
1419	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1420	/** Page directory (32-bit root). */
1421	#define PGMPOOL_IDX_PD 1
1422	/** Page Directory Pointer Table (PAE root). */
1423	#define PGMPOOL_IDX_PDPT 2
1424	/** AMD64 CR3 level index.*/
1425	#define PGMPOOL_IDX_AMD64_CR3 3
1426	/** Nested paging root.*/
1427	#define PGMPOOL_IDX_NESTED_ROOT 4
1428	/** The first normal index. */
1429	#define PGMPOOL_IDX_FIRST 5
1430	/** The last valid index. (inclusive, 14 bits) */
1431	#define PGMPOOL_IDX_LAST 0x3fff
1432	/** @} */
1433
1434	/** The NIL index for the parent chain. */
1435	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1436
1437	/**
1438	* Node in the chain linking a shadowed page to it's parent (user).
1439	*/
1440	#pragma pack(1)
1441	typedef struct PGMPOOLUSER
1442	{
1443	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1444	uint16_t iNext;
1445	/** The user page index. */
1446	uint16_t iUser;
1447	/** Index into the user table. */
1448	uint32_t iUserTable;
1449	} PGMPOOLUSER, *PPGMPOOLUSER;
1450	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1451	#pragma pack()
1452
1453
1454	/** The NIL index for the phys ext chain. */
1455	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1456
1457	/**
1458	* Node in the chain of physical cross reference extents.
1459	* @todo Calling this an 'extent' is not quite right, find a better name.
1460	*/
1461	#pragma pack(1)
1462	typedef struct PGMPOOLPHYSEXT
1463	{
1464	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1465	uint16_t iNext;
1466	/** The user page index. */
1467	uint16_t aidx[3];
1468	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1469	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1470	#pragma pack()
1471
1472
1473	/**
1474	* The kind of page that's being shadowed.
1475	*/
1476	typedef enum PGMPOOLKIND
1477	{
1478	/** The virtual invalid 0 entry. */
1479	PGMPOOLKIND_INVALID = 0,
1480	/** The entry is free (=unused). */
1481	PGMPOOLKIND_FREE,
1482
1483	/** Shw: 32-bit page table; Gst: no paging */
1484	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1485	/** Shw: 32-bit page table; Gst: 32-bit page table. */
1486	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1487	/** Shw: 32-bit page table; Gst: 4MB page. */
1488	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1489	/** Shw: PAE page table; Gst: no paging */
1490	PGMPOOLKIND_PAE_PT_FOR_PHYS,
1491	/** Shw: PAE page table; Gst: 32-bit page table. */
1492	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1493	/** Shw: PAE page table; Gst: Half of a 4MB page. */
1494	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1495	/** Shw: PAE page table; Gst: PAE page table. */
1496	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1497	/** Shw: PAE page table; Gst: 2MB page. */
1498	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1499
1500	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1501	PGMPOOLKIND_32BIT_PD,
1502	/** Shw: 32-bit page directory. Gst: no paging. */
1503	PGMPOOLKIND_32BIT_PD_PHYS,
1504	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1505	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1506	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1507	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1508	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1509	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1510	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1511	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1512	/** Shw: PAE page directory; Gst: PAE page directory. */
1513	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1514	/** Shw: PAE page directory; Gst: no paging. */
1515	PGMPOOLKIND_PAE_PD_PHYS,
1516
1517	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1518	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1519	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1520	PGMPOOLKIND_PAE_PDPT,
1521	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1522	PGMPOOLKIND_PAE_PDPT_PHYS,
1523
1524	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1525	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1526	/** Shw: 64-bit page directory pointer table; Gst: no paging */
1527	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1528	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1529	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1530	/** Shw: 64-bit page directory table; Gst: no paging */
1531	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
1532
1533	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1534	PGMPOOLKIND_64BIT_PML4,
1535
1536	/** Shw: EPT page directory pointer table; Gst: no paging */
1537	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1538	/** Shw: EPT page directory table; Gst: no paging */
1539	PGMPOOLKIND_EPT_PD_FOR_PHYS,
1540	/** Shw: EPT page table; Gst: no paging */
1541	PGMPOOLKIND_EPT_PT_FOR_PHYS,
1542
1543	/** Shw: Root Nested paging table. */
1544	PGMPOOLKIND_ROOT_NESTED,
1545
1546	/** The last valid entry. */
1547	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1548	} PGMPOOLKIND;
1549
1550
1551	/**
1552	* The tracking data for a page in the pool.
1553	*/
1554	typedef struct PGMPOOLPAGE
1555	{
1556	/** AVL node code with the (R3) physical address of this page. */
1557	AVLOHCPHYSNODECORE Core;
1558	/** Pointer to the R3 mapping of the page. */
1559	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1560	R3PTRTYPE(void *) pvPageR3;
1561	#else
1562	R3R0PTRTYPE(void *) pvPageR3;
1563	#endif
1564	/** The guest physical address. */
1565	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
1566	uint32_t Alignment0;
1567	#endif
1568	RTGCPHYS GCPhys;
1569	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1570	uint8_t enmKind;
1571	uint8_t bPadding;
1572	/** The index of this page. */
1573	uint16_t idx;
1574	/** The next entry in the list this page currently resides in.
1575	* It's either in the free list or in the GCPhys hash. */
1576	uint16_t iNext;
1577	#ifdef PGMPOOL_WITH_USER_TRACKING
1578	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1579	uint16_t iUserHead;
1580	/** The number of present entries. */
1581	uint16_t cPresent;
1582	/** The first entry in the table which is present. */
1583	uint16_t iFirstPresent;
1584	#endif
1585	#ifdef PGMPOOL_WITH_MONITORING
1586	/** The number of modifications to the monitored page. */
1587	uint16_t cModifications;
1588	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
1589	uint16_t iModifiedNext;
1590	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
1591	uint16_t iModifiedPrev;
1592	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1593	uint16_t iMonitoredNext;
1594	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1595	uint16_t iMonitoredPrev;
1596	#endif
1597	#ifdef PGMPOOL_WITH_CACHE
1598	/** The next page in the age list. */
1599	uint16_t iAgeNext;
1600	/** The previous page in the age list. */
1601	uint16_t iAgePrev;
1602	#endif /* PGMPOOL_WITH_CACHE */
1603	/** Used to indicate that the page is zeroed. */
1604	bool fZeroed;
1605	/** Used to indicate that a PT has non-global entries. */
1606	bool fSeenNonGlobal;
1607	/** Used to indicate that we're monitoring writes to the guest page. */
1608	bool fMonitored;
1609	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1610	* (All pages are in the age list.) */
1611	bool fCached;
1612	/** This is used by the R3 access handlers when invoked by an async thread.
1613	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1614	bool volatile fReusedFlushPending;
1615	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
1616	uint8_t cLocked;
1617	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
1618	/** Pointer to a const pool page. */
1619	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1620
1621
1622	#ifdef PGMPOOL_WITH_CACHE
1623	/** The hash table size. */
1624	# define PGMPOOL_HASH_SIZE 0x40
1625	/** The hash function. */
1626	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1627	#endif
1628
1629
1630	/**
1631	* The shadow page pool instance data.
1632	*
1633	* It's all one big allocation made at init time, except for the
1634	* pages that is. The user nodes follows immediatly after the
1635	* page structures.
1636	*/
1637	typedef struct PGMPOOL
1638	{
1639	/** The VM handle - R3 Ptr. */
1640	PVMR3 pVMR3;
1641	/** The VM handle - R0 Ptr. */
1642	PVMR0 pVMR0;
1643	/** The VM handle - RC Ptr. */
1644	PVMRC pVMRC;
1645	/** The max pool size. This includes the special IDs. */
1646	uint16_t cMaxPages;
1647	/** The current pool size. */
1648	uint16_t cCurPages;
1649	/** The head of the free page list. */
1650	uint16_t iFreeHead;
1651	/* Padding. */
1652	uint16_t u16Padding;
1653	#ifdef PGMPOOL_WITH_USER_TRACKING
1654	/** Head of the chain of free user nodes. */
1655	uint16_t iUserFreeHead;
1656	/** The number of user nodes we've allocated. */
1657	uint16_t cMaxUsers;
1658	/** The number of present page table entries in the entire pool. */
1659	uint32_t cPresent;
1660	/** Pointer to the array of user nodes - RC pointer. */
1661	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
1662	/** Pointer to the array of user nodes - R3 pointer. */
1663	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
1664	/** Pointer to the array of user nodes - R0 pointer. */
1665	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
1666	#endif /* PGMPOOL_WITH_USER_TRACKING */
1667	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1668	/** Head of the chain of free phys ext nodes. */
1669	uint16_t iPhysExtFreeHead;
1670	/** The number of user nodes we've allocated. */
1671	uint16_t cMaxPhysExts;
1672	/** Pointer to the array of physical xref extent - RC pointer. */
1673	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
1674	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
1675	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
1676	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
1677	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
1678	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1679	#ifdef PGMPOOL_WITH_CACHE
1680	/** Hash table for GCPhys addresses. */
1681	uint16_t aiHash[PGMPOOL_HASH_SIZE];
1682	/** The head of the age list. */
1683	uint16_t iAgeHead;
1684	/** The tail of the age list. */
1685	uint16_t iAgeTail;
1686	/** Set if the cache is enabled. */
1687	bool fCacheEnabled;
1688	#endif /* PGMPOOL_WITH_CACHE */
1689	#ifdef PGMPOOL_WITH_MONITORING
1690	/** Head of the list of modified pages. */
1691	uint16_t iModifiedHead;
1692	/** The current number of modified pages. */
1693	uint16_t cModifiedPages;
1694	/** Access handler, RC. */
1695	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
1696	/** Access handler, R0. */
1697	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
1698	/** Access handler, R3. */
1699	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
1700	/** The access handler description (HC ptr). */
1701	R3PTRTYPE(const char *) pszAccessHandler;
1702	#endif /* PGMPOOL_WITH_MONITORING */
1703	/** The number of pages currently in use. */
1704	uint16_t cUsedPages;
1705	#ifdef VBOX_WITH_STATISTICS
1706	/** The high wather mark for cUsedPages. */
1707	uint16_t cUsedPagesHigh;
1708	uint32_t Alignment1; /*< Align the next member on a 64-bit boundrary. /
1709	/** Profiling pgmPoolAlloc(). */
1710	STAMPROFILEADV StatAlloc;
1711	/** Profiling pgmPoolClearAll(). */
1712	STAMPROFILE StatClearAll;
1713	/** Profiling pgmPoolFlushAllInt(). */
1714	STAMPROFILE StatFlushAllInt;
1715	/** Profiling pgmPoolFlushPage(). */
1716	STAMPROFILE StatFlushPage;
1717	/** Profiling pgmPoolFree(). */
1718	STAMPROFILE StatFree;
1719	/** Profiling time spent zeroing pages. */
1720	STAMPROFILE StatZeroPage;
1721	# ifdef PGMPOOL_WITH_USER_TRACKING
1722	/** Profiling of pgmPoolTrackDeref. */
1723	STAMPROFILE StatTrackDeref;
1724	/** Profiling pgmTrackFlushGCPhysPT. */
1725	STAMPROFILE StatTrackFlushGCPhysPT;
1726	/** Profiling pgmTrackFlushGCPhysPTs. */
1727	STAMPROFILE StatTrackFlushGCPhysPTs;
1728	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
1729	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
1730	/** Number of times we've been out of user records. */
1731	STAMCOUNTER StatTrackFreeUpOneUser;
1732	# endif
1733	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1734	/** Profiling deref activity related tracking GC physical pages. */
1735	STAMPROFILE StatTrackDerefGCPhys;
1736	/** Number of linear searches for a HCPhys in the ram ranges. */
1737	STAMCOUNTER StatTrackLinearRamSearches;
1738	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
1739	STAMCOUNTER StamTrackPhysExtAllocFailures;
1740	# endif
1741	# ifdef PGMPOOL_WITH_MONITORING
1742	/** Profiling the RC/R0 access handler. */
1743	STAMPROFILE StatMonitorRZ;
1744	/** Times we've failed interpreting the instruction. */
1745	STAMCOUNTER StatMonitorRZEmulateInstr;
1746	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
1747	STAMPROFILE StatMonitorRZFlushPage;
1748	/** Times we've detected fork(). */
1749	STAMCOUNTER StatMonitorRZFork;
1750	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
1751	STAMPROFILE StatMonitorRZHandled;
1752	/** Times we've failed interpreting a patch code instruction. */
1753	STAMCOUNTER StatMonitorRZIntrFailPatch1;
1754	/** Times we've failed interpreting a patch code instruction during flushing. */
1755	STAMCOUNTER StatMonitorRZIntrFailPatch2;
1756	/** The number of times we've seen rep prefixes we can't handle. */
1757	STAMCOUNTER StatMonitorRZRepPrefix;
1758	/** Profiling the REP STOSD cases we've handled. */
1759	STAMPROFILE StatMonitorRZRepStosd;
1760
1761	/** Profiling the R3 access handler. */
1762	STAMPROFILE StatMonitorR3;
1763	/** Times we've failed interpreting the instruction. */
1764	STAMCOUNTER StatMonitorR3EmulateInstr;
1765	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
1766	STAMPROFILE StatMonitorR3FlushPage;
1767	/** Times we've detected fork(). */
1768	STAMCOUNTER StatMonitorR3Fork;
1769	/** Profiling the R3 access we've handled (except REP STOSD). */
1770	STAMPROFILE StatMonitorR3Handled;
1771	/** The number of times we've seen rep prefixes we can't handle. */
1772	STAMCOUNTER StatMonitorR3RepPrefix;
1773	/** Profiling the REP STOSD cases we've handled. */
1774	STAMPROFILE StatMonitorR3RepStosd;
1775	/** The number of times we're called in an async thread an need to flush. */
1776	STAMCOUNTER StatMonitorR3Async;
1777	/** The high wather mark for cModifiedPages. */
1778	uint16_t cModifiedPagesHigh;
1779	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundrary. /
1780	# endif
1781	# ifdef PGMPOOL_WITH_CACHE
1782	/** The number of cache hits. */
1783	STAMCOUNTER StatCacheHits;
1784	/** The number of cache misses. */
1785	STAMCOUNTER StatCacheMisses;
1786	/** The number of times we've got a conflict of 'kind' in the cache. */
1787	STAMCOUNTER StatCacheKindMismatches;
1788	/** Number of times we've been out of pages. */
1789	STAMCOUNTER StatCacheFreeUpOne;
1790	/** The number of cacheable allocations. */
1791	STAMCOUNTER StatCacheCacheable;
1792	/** The number of uncacheable allocations. */
1793	STAMCOUNTER StatCacheUncacheable;
1794	# endif
1795	#elif HC_ARCH_BITS == 64
1796	uint32_t Alignment3; /*< Align the next member on a 64-bit boundrary. /
1797	#endif
1798	/** The AVL tree for looking up a page by its HC physical address. */
1799	AVLOHCPHYSTREE HCPhysTree;
1800	uint32_t Alignment4; /*< Align the next member on a 64-bit boundrary. /
1801	/** Array of pages. (cMaxPages in length)
1802	* The Id is the index into thist array.
1803	*/
1804	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
1805	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
1806
1807
1808	/** @def PGMPOOL_PAGE_2_PTR
1809	* Maps a pool page pool into the current context.
1810	*
1811	* @returns VBox status code.
1812	* @param pVM The VM handle.
1813	* @param pPage The pool page.
1814	*
1815	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1816	* small page window employeed by that function. Be careful.
1817	* @remark There is no need to assert on the result.
1818	*/
1819	#if defined(IN_RC)
1820	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1821	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1822	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1823	#elif defined(VBOX_STRICT)
1824	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageStrict(pPage)
1825	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
1826	{
1827	Assert(pPage && pPage->pvPageR3);
1828	return pPage->pvPageR3;
1829	}
1830	#else
1831	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) ((pPage)->pvPageR3)
1832	#endif
1833
1834	/** @def PGMPOOL_PAGE_2_PTR_BY_PGM
1835	* Maps a pool page pool into the current context.
1836	*
1837	* @returns VBox status code.
1838	* @param pPGM Pointer to the PGM instance data.
1839	* @param pPage The pool page.
1840	*
1841	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1842	* small page window employeed by that function. Be careful.
1843	* @remark There is no need to assert on the result.
1844	*/
1845	#if defined(IN_RC)
1846	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1847	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1848	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1849	#else
1850	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGM2VM(pPGM), pPage)
1851	#endif
1852
1853	/** @def PGMPOOL_PAGE_2_PTR_BY_PGMCPU
1854	* Maps a pool page pool into the current context.
1855	*
1856	* @returns VBox status code.
1857	* @param pPGM Pointer to the PGMCPU instance data.
1858	* @param pPage The pool page.
1859	*
1860	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1861	* small page window employeed by that function. Be careful.
1862	* @remark There is no need to assert on the result.
1863	*/
1864	#if defined(IN_RC)
1865	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1866	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1867	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1868	#else
1869	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGMCPU2VM(pPGM), pPage)
1870	#endif
1871
1872
1873	/** @name Per guest page tracking data.
1874	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
1875	* is to use more bits for it and split it up later on. But for now we'll play
1876	* safe and change as little as possible.
1877	*
1878	* The 16-bit word has two parts:
1879	*
1880	* The first 14-bit forms the @a idx field. It is either the index of a page in
1881	* the shadow page pool, or and index into the extent list.
1882	*
1883	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
1884	* shadow page pool references to the page. If cRefs equals
1885	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
1886	* (misnomer) table and not the shadow page pool.
1887	*
1888	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
1889	* the 16-bit word.
1890	*
1891	* @{ */
1892	/** The shift count for getting to the cRefs part. */
1893	#define PGMPOOL_TD_CREFS_SHIFT 14
1894	/** The mask applied after shifting the tracking data down by
1895	* PGMPOOL_TD_CREFS_SHIFT. */
1896	#define PGMPOOL_TD_CREFS_MASK 0x3
1897	/** The cRef value used to indiciate that the idx is the head of a
1898	* physical cross reference list. */
1899	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
1900	/** The shift used to get idx. */
1901	#define PGMPOOL_TD_IDX_SHIFT 0
1902	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
1903	#define PGMPOOL_TD_IDX_MASK 0x3fff
1904	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
1905	* simply too many mappings of this page. */
1906	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
1907
1908	/** @def PGMPOOL_TD_MAKE
1909	* Makes a 16-bit tracking data word.
1910	*
1911	* @returns tracking data.
1912	* @param cRefs The @a cRefs field. Must be within bounds!
1913	* @param idx The @a idx field. Must also be within bounds! */
1914	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
1915
1916	/** @def PGMPOOL_TD_GET_CREFS
1917	* Get the @a cRefs field from a tracking data word.
1918	*
1919	* @returns The @a cRefs field
1920	* @param u16 The tracking data word. */
1921	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
1922
1923	/** @def PGMPOOL_TD_GET_IDX
1924	* Get the @a idx field from a tracking data word.
1925	*
1926	* @returns The @a idx field
1927	* @param u16 The tracking data word. */
1928	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
1929	/** @} */
1930
1931
1932	/**
1933	* Trees are using self relative offsets as pointers.
1934	* So, all its data, including the root pointer, must be in the heap for HC and GC
1935	* to have the same layout.
1936	*/
1937	typedef struct PGMTREES
1938	{
1939	/** Physical access handlers (AVL range+offsetptr tree). */
1940	AVLROGCPHYSTREE PhysHandlers;
1941	/** Virtual access handlers (AVL range + GC ptr tree). */
1942	AVLROGCPTRTREE VirtHandlers;
1943	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
1944	AVLROGCPHYSTREE PhysToVirtHandlers;
1945	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
1946	AVLROGCPTRTREE HyperVirtHandlers;
1947	} PGMTREES;
1948	/** Pointer to PGM trees. */
1949	typedef PGMTREES *PPGMTREES;
1950
1951
1952	/** @name Paging mode macros
1953	* @{ */
1954	#ifdef IN_RC
1955	# define PGM_CTX(a,b) a##RC##b
1956	# define PGM_CTX_STR(a,b) a "GC" b
1957	# define PGM_CTX_DECL(type) VMMRCDECL(type)
1958	#else
1959	# ifdef IN_RING3
1960	# define PGM_CTX(a,b) a##R3##b
1961	# define PGM_CTX_STR(a,b) a "R3" b
1962	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
1963	# else
1964	# define PGM_CTX(a,b) a##R0##b
1965	# define PGM_CTX_STR(a,b) a "R0" b
1966	# define PGM_CTX_DECL(type) VMMDECL(type)
1967	# endif
1968	#endif
1969
1970	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
1971	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
1972	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
1973	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
1974	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
1975	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
1976	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
1977	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
1978	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
1979	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
1980	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
1981	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
1982	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
1983	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
1984	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
1985	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
1986	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
1987
1988	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
1989	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
1990	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
1991	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
1992	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
1993	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
1994	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
1995	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
1996	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
1997	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
1998	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
1999	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2000	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2001	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2002	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2003	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2004	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2005
2006	/* Shw_Gst */
2007	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2008	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2009	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2010	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2011	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2012	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2013	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2014	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2015	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2016	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2017	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2018	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2019	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2020	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2021	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2022	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2023	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2024	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2025	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2026
2027	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2028	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2029	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2030	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2031	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2032	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2033	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2034	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2035	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2036	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2037	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2038	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2039	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2040	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2041	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2042	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2043	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2044	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2045	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2046	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2047	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2048	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2049	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2050	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2051	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2052	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2053	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2054	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2055	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2056	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2057	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2058	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2059	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2060	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2061	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2062	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2063	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2064
2065	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2066	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2067	/** @} */
2068
2069	/**
2070	* Data for each paging mode.
2071	*/
2072	typedef struct PGMMODEDATA
2073	{
2074	/** The guest mode type. */
2075	uint32_t uGstType;
2076	/** The shadow mode type. */
2077	uint32_t uShwType;
2078
2079	/** @name Function pointers for Shadow paging.
2080	* @{
2081	*/
2082	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2083	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2084	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2085	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2086
2087	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2088	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2089
2090	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2091	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2092	/** @} */
2093
2094	/** @name Function pointers for Guest paging.
2095	* @{
2096	*/
2097	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2098	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2099	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2100	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2101	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2102	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2103	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2104	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2105	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2106	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2107	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2108	/** @} */
2109
2110	/** @name Function pointers for Both Shadow and Guest paging.
2111	* @{
2112	*/
2113	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2114	/* no pfnR3BthTrap0eHandler */
2115	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2116	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2117	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2118	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2119	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2120	#ifdef VBOX_STRICT
2121	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2122	#endif
2123	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2124	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2125
2126	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2127	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2128	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2129	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2130	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2131	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2132	#ifdef VBOX_STRICT
2133	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2134	#endif
2135	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2136	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2137
2138	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2139	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2140	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2141	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2142	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2143	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2144	#ifdef VBOX_STRICT
2145	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2146	#endif
2147	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2148	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2149	/** @} */
2150	} PGMMODEDATA, *PPGMMODEDATA;
2151
2152
2153
2154	/**
2155	* Converts a PGM pointer into a VM pointer.
2156	* @returns Pointer to the VM structure the PGM is part of.
2157	* @param pPGM Pointer to PGM instance data.
2158	*/
2159	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2160
2161	/**
2162	* PGM Data (part of VM)
2163	*/
2164	typedef struct PGM
2165	{
2166	/** Offset to the VM structure. */
2167	RTINT offVM;
2168	/** Offset of the PGMCPU structure relative to VMCPU. */
2169	RTINT offVCpuPGM;
2170
2171	/** @cfgm{RamPreAlloc, boolean, false}
2172	* Indicates whether the base RAM should all be allocated before starting
2173	* the VM (default), or if it should be allocated when first written to.
2174	*/
2175	bool fRamPreAlloc;
2176	/** Alignment padding. */
2177	bool afAlignment0[7];
2178
2179	/** What needs syncing (PGM_SYNC_*).
2180	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2181	* PGMFlushTLB, and PGMR3Load. */
2182	RTUINT fGlobalSyncFlags;
2183
2184	/*
2185	* This will be redefined at least two more times before we're done, I'm sure.
2186	* The current code is only to get on with the coding.
2187	* - 2004-06-10: initial version, bird.
2188	* - 2004-07-02: 1st time, bird.
2189	* - 2004-10-18: 2nd time, bird.
2190	* - 2005-07-xx: 3rd time, bird.
2191	*/
2192
2193	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2194	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
2195	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2196	RCPTRTYPE(PX86PTEPAE) paDynPageMapPaePTEsGC;
2197
2198	/** The host paging mode. (This is what SUPLib reports.) */
2199	SUPPAGINGMODE enmHostMode;
2200
2201	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2202	RTGCPHYS GCPhys4MBPSEMask;
2203
2204	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2205	* This is sorted by physical address and contains no overlapping ranges. */
2206	R3PTRTYPE(PPGMRAMRANGE) pRamRangesR3;
2207	/** R0 pointer corresponding to PGM::pRamRangesR3. */
2208	R0PTRTYPE(PPGMRAMRANGE) pRamRangesR0;
2209	/** RC pointer corresponding to PGM::pRamRangesR3. */
2210	RCPTRTYPE(PPGMRAMRANGE) pRamRangesRC;
2211	RTRCPTR alignment4; /*< structure alignment. /
2212
2213	/** Pointer to the list of ROM ranges - for R3.
2214	* This is sorted by physical address and contains no overlapping ranges. */
2215	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2216	/** R0 pointer corresponding to PGM::pRomRangesR3. */
2217	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2218	/** RC pointer corresponding to PGM::pRomRangesR3. */
2219	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
2220	/** Alignment padding. */
2221	RTRCPTR GCPtrPadding2;
2222
2223	/** Pointer to the list of MMIO2 ranges - for R3.
2224	* Registration order. */
2225	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
2226
2227	/** PGM offset based trees - R3 Ptr. */
2228	R3PTRTYPE(PPGMTREES) pTreesR3;
2229	/** PGM offset based trees - R0 Ptr. */
2230	R0PTRTYPE(PPGMTREES) pTreesR0;
2231	/** PGM offset based trees - RC Ptr. */
2232	RCPTRTYPE(PPGMTREES) pTreesRC;
2233
2234	/** Linked list of GC mappings - for RC.
2235	* The list is sorted ascending on address.
2236	*/
2237	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
2238	/** Linked list of GC mappings - for HC.
2239	* The list is sorted ascending on address.
2240	*/
2241	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
2242	/** Linked list of GC mappings - for R0.
2243	* The list is sorted ascending on address.
2244	*/
2245	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
2246
2247	/** Pointer to the 5 page CR3 content mapping.
2248	* The first page is always the CR3 (in some form) while the 4 other pages
2249	* are used of the PDs in PAE mode. */
2250	RTGCPTR GCPtrCR3Mapping;
2251	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
2252	uint32_t u32Alignment;
2253	#endif
2254
2255	/** Indicates that PGMR3FinalizeMappings has been called and that further
2256	* PGMR3MapIntermediate calls will be rejected. */
2257	bool fFinalizedMappings;
2258	/** If set no conflict checks are required. (boolean) */
2259	bool fMappingsFixed;
2260	/** If set, then no mappings are put into the shadow page table. (boolean) */
2261	bool fDisableMappings;
2262	/** Size of fixed mapping */
2263	uint32_t cbMappingFixed;
2264	/** Base address (GC) of fixed mapping */
2265	RTGCPTR GCPtrMappingFixed;
2266	/** The address of the previous RAM range mapping. */
2267	RTGCPTR GCPtrPrevRamRangeMapping;
2268
2269	/** @name Intermediate Context
2270	* @{ */
2271	/** Pointer to the intermediate page directory - Normal. */
2272	R3PTRTYPE(PX86PD) pInterPD;
2273	/** Pointer to the intermedate page tables - Normal.
2274	* There are two page tables, one for the identity mapping and one for
2275	* the host context mapping (of the core code). */
2276	R3PTRTYPE(PX86PT) apInterPTs[2];
2277	/** Pointer to the intermedate page tables - PAE. */
2278	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
2279	/** Pointer to the intermedate page directory - PAE. */
2280	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
2281	/** Pointer to the intermedate page directory - PAE. */
2282	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
2283	/** Pointer to the intermedate page-map level 4 - AMD64. */
2284	R3PTRTYPE(PX86PML4) pInterPaePML4;
2285	/** Pointer to the intermedate page directory - AMD64. */
2286	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
2287	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
2288	RTHCPHYS HCPhysInterPD;
2289	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
2290	RTHCPHYS HCPhysInterPaePDPT;
2291	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
2292	RTHCPHYS HCPhysInterPaePML4;
2293	/** @} */
2294
2295	/** Base address of the dynamic page mapping area.
2296	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
2297	*/
2298	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
2299	/** The index of the last entry used in the dynamic page mapping area. */
2300	RTUINT iDynPageMapLast;
2301	/** Cache containing the last entries in the dynamic page mapping area.
2302	* The cache size is covering half of the mapping area. */
2303	RTHCPHYS aHCPhysDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT + 1)];
2304	/** Keep a lock counter for the full (!) mapping area. */
2305	uint32_t aLockedDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT)];
2306
2307	/** The address of the ring-0 mapping cache if we're making use of it. */
2308	RTR0PTR pvR0DynMapUsed;
2309
2310	/** PGM critical section.
2311	* This protects the physical & virtual access handlers, ram ranges,
2312	* and the page flag updating (some of it anyway).
2313	*/
2314	PDMCRITSECT CritSect;
2315
2316	/** Pointer to SHW+GST mode data (function pointers).
2317	* The index into this table is made up from */
2318	R3PTRTYPE(PPGMMODEDATA) paModeData;
2319
2320	/** Shadow Page Pool - R3 Ptr. */
2321	R3PTRTYPE(PPGMPOOL) pPoolR3;
2322	/** Shadow Page Pool - R0 Ptr. */
2323	R0PTRTYPE(PPGMPOOL) pPoolR0;
2324	/** Shadow Page Pool - RC Ptr. */
2325	RCPTRTYPE(PPGMPOOL) pPoolRC;
2326
2327	/** We're not in a state which permits writes to guest memory.
2328	* (Only used in strict builds.) */
2329	bool fNoMorePhysWrites;
2330
2331	/** Flush the cache on the next access. */
2332	bool fPhysCacheFlushPending;
2333	/** @todo r=bird: Fix member names!*/
2334	/** PGMPhysRead cache */
2335	PGMPHYSCACHE pgmphysreadcache;
2336	/** PGMPhysWrite cache */
2337	PGMPHYSCACHE pgmphyswritecache;
2338
2339	/**
2340	* Data associated with managing the ring-3 mappings of the allocation chunks.
2341	*/
2342	struct
2343	{
2344	/** The chunk tree, ordered by chunk id. */
2345	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2346	R3PTRTYPE(PAVLU32NODECORE) pTree;
2347	#else
2348	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
2349	#endif
2350	/** The chunk mapping TLB. */
2351	PGMCHUNKR3MAPTLB Tlb;
2352	/** The number of mapped chunks. */
2353	uint32_t c;
2354	/** The maximum number of mapped chunks.
2355	* @cfgm PGM/MaxRing3Chunks */
2356	uint32_t cMax;
2357	/** The chunk age tree, ordered by ageing sequence number. */
2358	R3PTRTYPE(PAVLLU32NODECORE) pAgeTree;
2359	/** The current time. */
2360	uint32_t iNow;
2361	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
2362	uint32_t AgeingCountdown;
2363	} ChunkR3Map;
2364
2365	/**
2366	* The page mapping TLB for ring-3 and (for the time being) ring-0.
2367	*/
2368	PGMPAGER3MAPTLB PhysTlbHC;
2369
2370	/** @name The zero page.
2371	* @{ */
2372	/** The host physical address of the zero page. */
2373	RTHCPHYS HCPhysZeroPg;
2374	/** The ring-3 mapping of the zero page. */
2375	RTR3PTR pvZeroPgR3;
2376	/** The ring-0 mapping of the zero page. */
2377	RTR0PTR pvZeroPgR0;
2378	/** The GC mapping of the zero page. */
2379	RTGCPTR pvZeroPgRC;
2380	#if GC_ARCH_BITS != 32
2381	uint32_t u32ZeroAlignment; /*< Alignment padding. /
2382	#endif
2383	/** @}*/
2384
2385	/** The number of handy pages. */
2386	uint32_t cHandyPages;
2387	/**
2388	* Array of handy pages.
2389	*
2390	* This array is used in a two way communication between pgmPhysAllocPage
2391	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
2392	* an intermediary.
2393	*
2394	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
2395	* (The current size of 32 pages, means 128 KB of handy memory.)
2396	*/
2397	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
2398
2399	/** @name Error injection.
2400	* @{ */
2401	/** Inject handy page allocation errors pretending we're completely out of
2402	* memory. */
2403	bool volatile fErrInjHandyPages;
2404	/** Padding. */
2405	bool afReserved[7];
2406	/** @} */
2407
2408	/** @name Release Statistics
2409	* @{ */
2410	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero.) /
2411	uint32_t cPrivatePages; /*< The number of private pages. /
2412	uint32_t cSharedPages; /*< The number of shared pages. /
2413	uint32_t cZeroPages; /*< The number of zero backed pages. /
2414
2415	/** The number of times we were forced to change the hypervisor region location. */
2416	STAMCOUNTER cRelocations;
2417	/** @} */
2418
2419	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2420	/* R3 only: */
2421	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2422	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2423
2424	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2425	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2426	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2427	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2428	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2429	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2430	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2431	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2432	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2433	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2434	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2435	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2436	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2437	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2438	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2439	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2440	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2441	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2442	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2443	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2444	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2445	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2446
2447	/* RC only: */
2448	STAMCOUNTER StatRCDynMapCacheMisses; /*< RC: The number of dynamic page mapping cache misses /
2449	STAMCOUNTER StatRCDynMapCacheHits; /*< RC: The number of dynamic page mapping cache hits /
2450	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2451	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2452
2453	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2454	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2455	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2456	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2457	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2458	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2459	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2460	# endif
2461	#endif
2462	} PGM;
2463	/** Pointer to the PGM instance data. */
2464	typedef PGM *PPGM;
2465
2466
2467	/**
2468	* Converts a PGMCPU pointer into a VM pointer.
2469	* @returns Pointer to the VM structure the PGM is part of.
2470	* @param pPGM Pointer to PGMCPU instance data.
2471	*/
2472	#define PGMCPU2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2473
2474	/**
2475	* Converts a PGMCPU pointer into a PGM pointer.
2476	* @returns Pointer to the VM structure the PGM is part of.
2477	* @param pPGM Pointer to PGMCPU instance data.
2478	*/
2479	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char*)pPGMCpu - pPGMCpu->offPGM) )
2480
2481	/**
2482	* PGMCPU Data (part of VMCPU).
2483	*/
2484	typedef struct PGMCPU
2485	{
2486	/** Offset to the VM structure. */
2487	RTINT offVM;
2488	/** Offset to the VMCPU structure. */
2489	RTINT offVCpu;
2490	/** Offset of the PGM structure relative to VMCPU. */
2491	RTINT offPGM;
2492	RTINT uPadding0; /*< structure size alignment. /
2493
2494	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2495	/** Automatically tracked physical memory mapping set.
2496	* Ring-0 and strict raw-mode builds. */
2497	PGMMAPSET AutoSet;
2498	#endif
2499
2500	/** A20 gate mask.
2501	* Our current approach to A20 emulation is to let REM do it and don't bother
2502	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2503	* But whould need arrise, we'll subject physical addresses to this mask. */
2504	RTGCPHYS GCPhysA20Mask;
2505	/** A20 gate state - boolean! */
2506	bool fA20Enabled;
2507
2508	/** What needs syncing (PGM_SYNC_*).
2509	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2510	* PGMFlushTLB, and PGMR3Load. */
2511	RTUINT fSyncFlags;
2512
2513	/** The shadow paging mode. */
2514	PGMMODE enmShadowMode;
2515	/** The guest paging mode. */
2516	PGMMODE enmGuestMode;
2517
2518	/** The current physical address representing in the guest CR3 register. */
2519	RTGCPHYS GCPhysCR3;
2520
2521	/** @name 32-bit Guest Paging.
2522	* @{ */
2523	/** The guest's page directory, R3 pointer. */
2524	R3PTRTYPE(PX86PD) pGst32BitPdR3;
2525	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2526	/** The guest's page directory, R0 pointer. */
2527	R0PTRTYPE(PX86PD) pGst32BitPdR0;
2528	#endif
2529	/** The guest's page directory, static RC mapping. */
2530	RCPTRTYPE(PX86PD) pGst32BitPdRC;
2531	/** @} */
2532
2533	/** @name PAE Guest Paging.
2534	* @{ */
2535	/** The guest's page directory pointer table, static RC mapping. */
2536	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
2537	/** The guest's page directory pointer table, R3 pointer. */
2538	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
2539	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2540	/** The guest's page directory pointer table, R0 pointer. */
2541	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
2542	#endif
2543
2544	/** The guest's page directories, R3 pointers.
2545	* These are individual pointers and don't have to be adjecent.
2546	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2547	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
2548	/** The guest's page directories, R0 pointers.
2549	* Same restrictions as apGstPaePDsR3. */
2550	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2551	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
2552	#endif
2553	/** The guest's page directories, static GC mapping.
2554	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
2555	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2556	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
2557	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
2558	RTGCPHYS aGCPhysGstPaePDs[4];
2559	/** The physical addresses of the monitored guest page directories (PAE). */
2560	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
2561	/** @} */
2562
2563	/** @name AMD64 Guest Paging.
2564	* @{ */
2565	/** The guest's page directory pointer table, R3 pointer. */
2566	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
2567	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2568	/** The guest's page directory pointer table, R0 pointer. */
2569	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
2570	#endif
2571	/** @} */
2572
2573	/** Pointer to the page of the current active CR3 - R3 Ptr. */
2574	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
2575	/** Pointer to the page of the current active CR3 - R0 Ptr. */
2576	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
2577	/** Pointer to the page of the current active CR3 - RC Ptr. */
2578	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
2579	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
2580	uint32_t iShwUser;
2581	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
2582	uint32_t iShwUserTable;
2583	# if HC_ARCH_BITS == 64
2584	RTRCPTR alignment6; /*< structure size alignment. /
2585	# endif
2586	/** @} */
2587
2588	/** @name Function pointers for Shadow paging.
2589	* @{
2590	*/
2591	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2592	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2593	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2594	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2595
2596	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2597	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2598
2599	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2600	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2601
2602	/** @} */
2603
2604	/** @name Function pointers for Guest paging.
2605	* @{
2606	*/
2607	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2608	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2609	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2610	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2611	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2612	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2613	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2614	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2615	#if HC_ARCH_BITS == 64
2616	RTRCPTR alignment3; /*< structure size alignment. /
2617	#endif
2618
2619	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2620	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2621	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2622	/** @} */
2623
2624	/** @name Function pointers for Both Shadow and Guest paging.
2625	* @{
2626	*/
2627	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2628	/* no pfnR3BthTrap0eHandler */
2629	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2630	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2631	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2632	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2633	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2634	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2635	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2636	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2637
2638	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2639	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2640	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2641	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2642	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2643	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2644	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2645	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2646	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2647
2648	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2649	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2650	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2651	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2652	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2653	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2654	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2655	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2656	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2657	#if HC_ARCH_BITS == 64
2658	RTRCPTR alignment2; /*< structure size alignment. /
2659	#endif
2660	/** @} */
2661
2662	/** @name Release Statistics
2663	* @{ */
2664	/** The number of times the guest has switched mode since last reset or statistics reset. */
2665	STAMCOUNTER cGuestModeChanges;
2666	/** @} */
2667
2668	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2669	/** @name Statistics
2670	* @{ */
2671	/** RC: Which statistic this \#PF should be attributed to. */
2672	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
2673	RTRCPTR padding0;
2674	/** R0: Which statistic this \#PF should be attributed to. */
2675	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
2676	RTR0PTR padding1;
2677
2678	/* Common */
2679	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
2680	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
2681
2682	/* R0 only: */
2683	STAMCOUNTER StatR0DynMapMigrateInvlPg; /*< R0: invlpg in PGMDynMapMigrateAutoSet. /
2684	STAMPROFILE StatR0DynMapGCPageInl; /*< R0: Calls to pgmR0DynMapGCPageInlined. /
2685	STAMCOUNTER StatR0DynMapGCPageInlHits; /*< R0: Hash table lookup hits. /
2686	STAMCOUNTER StatR0DynMapGCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2687	STAMCOUNTER StatR0DynMapGCPageInlRamHits; /*< R0: 1st ram range hits. /
2688	STAMCOUNTER StatR0DynMapGCPageInlRamMisses; /*< R0: 1st ram range misses, takes slow path. /
2689	STAMPROFILE StatR0DynMapHCPageInl; /*< R0: Calls to pgmR0DynMapHCPageInlined. /
2690	STAMCOUNTER StatR0DynMapHCPageInlHits; /*< R0: Hash table lookup hits. /
2691	STAMCOUNTER StatR0DynMapHCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2692	STAMPROFILE StatR0DynMapHCPage; /*< R0: Calls to PGMDynMapHCPage. /
2693	STAMCOUNTER StatR0DynMapSetOptimize; /*< R0: Calls to pgmDynMapOptimizeAutoSet. /
2694	STAMCOUNTER StatR0DynMapSetSearchFlushes; /*< R0: Set search restorting to subset flushes. /
2695	STAMCOUNTER StatR0DynMapSetSearchHits; /*< R0: Set search hits. /
2696	STAMCOUNTER StatR0DynMapSetSearchMisses; /*< R0: Set search misses. /
2697	STAMCOUNTER StatR0DynMapPage; /*< R0: Calls to pgmR0DynMapPage. /
2698	STAMCOUNTER StatR0DynMapPageHits0; /*< R0: Hits at iPage+0. /
2699	STAMCOUNTER StatR0DynMapPageHits1; /*< R0: Hits at iPage+1. /
2700	STAMCOUNTER StatR0DynMapPageHits2; /*< R0: Hits at iPage+2. /
2701	STAMCOUNTER StatR0DynMapPageInvlPg; /*< R0: invlpg. /
2702	STAMCOUNTER StatR0DynMapPageSlow; /*< R0: Calls to pgmR0DynMapPageSlow. /
2703	STAMCOUNTER StatR0DynMapPageSlowLoopHits; /*< R0: Hits in the pgmR0DynMapPageSlow search loop. /
2704	STAMCOUNTER StatR0DynMapPageSlowLoopMisses; /*< R0: Misses in the pgmR0DynMapPageSlow search loop. /
2705	//STAMCOUNTER StatR0DynMapPageSlowLostHits; /*< R0: Lost hits. /
2706	STAMCOUNTER StatR0DynMapSubsets; /*< R0: Times PGMDynMapPushAutoSubset was called. /
2707	STAMCOUNTER StatR0DynMapPopFlushes; /*< R0: Times PGMDynMapPopAutoSubset flushes the subset. /
2708	STAMCOUNTER aStatR0DynMapSetSize[11]; /*< R0: Set size distribution. /
2709
2710	/* RZ only: */
2711	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
2712	STAMPROFILE StatRZTrap0eTimeCheckPageFault;
2713	STAMPROFILE StatRZTrap0eTimeSyncPT;
2714	STAMPROFILE StatRZTrap0eTimeMapping;
2715	STAMPROFILE StatRZTrap0eTimeOutOfSync;
2716	STAMPROFILE StatRZTrap0eTimeHandlers;
2717	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
2718	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
2719	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
2720	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
2721	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
2722	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
2723	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
2724	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
2725	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
2726	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
2727	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
2728	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
2729	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
2730	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
2731	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
2732	STAMCOUNTER StatRZTrap0eHandlersPhysical; /*< RC/R0: Number of traps due to physical access handlers. /
2733	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
2734	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
2735	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
2736	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
2737	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
2738	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: #PF err kind /
2739	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: #PF err kind /
2740	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: #PF err kind /
2741	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: #PF err kind /
2742	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: #PF err kind /
2743	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: #PF err kind /
2744	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: #PF err kind /
2745	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: #PF err kind /
2746	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: #PF err kind /
2747	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: #PF err kind /
2748	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: #PF err kind /
2749	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest #PFs. /
2750	STAMCOUNTER StatRZTrap0eGuestPFUnh; /*< RC/R0: Real guest #PF ending up at the end of the #PF code. /
2751	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest #PF to HMA or other mapping. /
2752	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
2753	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
2754	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the #PFs. /
2755	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
2756	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
2757	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
2758	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
2759	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
2760
2761	/* HC - R3 and (maybe) R0: */
2762
2763	/* RZ & R3: */
2764	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
2765	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
2766	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
2767	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
2768	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
2769	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
2770	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
2771	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
2772	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
2773	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
2774	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
2775	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
2776	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
2777	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
2778	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2779	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
2780	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
2781	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
2782	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2783	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2784	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
2785	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
2786	STAMCOUNTER StatRZDirtyPageStale; /*< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) /
2787	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
2788	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
2789	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
2790	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
2791	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
2792	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
2793	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
2794	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2795	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2796	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
2797	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2798	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2799	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2800	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2801	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
2802	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
2803	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2804	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2805	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2806	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2807	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
2808
2809	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
2810	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
2811	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
2812	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
2813	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
2814	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
2815	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
2816	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
2817	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
2818	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
2819	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
2820	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
2821	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
2822	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
2823	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2824	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
2825	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
2826	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
2827	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2828	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2829	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
2830	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
2831	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
2832	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
2833	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
2834	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
2835	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
2836	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
2837	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
2838	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2839	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
2840	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2841	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2842	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2843	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2844	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2845	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
2846	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
2847	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2848	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2849	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2850	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2851	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
2852	/** @} */
2853	#endif /* VBOX_WITH_STATISTICS */
2854	} PGMCPU;
2855	/** Pointer to the per-cpu PGM data. */
2856	typedef PGMCPU *PPGMCPU;
2857
2858
2859	/** @name PGM::fSyncFlags Flags
2860	* @{
2861	*/
2862	/** Updates the virtual access handler state bit in PGMPAGE. */
2863	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
2864	/** Always sync CR3. */
2865	#define PGM_SYNC_ALWAYS RT_BIT(1)
2866	/** Check monitoring on next CR3 (re)load and invalidate page.
2867	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
2868	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
2869	/** Check guest mapping in SyncCR3. */
2870	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
2871	/** Clear the page pool (a light weight flush). */
2872	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT 8
2873	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT)
2874	/** @} */
2875
2876
2877	__BEGIN_DECLS
2878
2879	int pgmLock(PVM pVM);
2880	void pgmUnlock(PVM pVM);
2881
2882	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
2883	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
2884	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
2885	void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping);
2886	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2887
2888	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
2889	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
2890	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage);
2891	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
2892	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
2893	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
2894	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
2895	#else
2896	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
2897	#endif
2898	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2899
2900
2901	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2902	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
2903	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2904	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2905	int pgmPhysPageMakeWritableUnlocked(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2906	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv);
2907	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
2908	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
2909	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
2910	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
2911	#ifdef IN_RING3
2912	void pgmR3PhysRelinkRamRanges(PVM pVM);
2913	int pgmR3PhysRamPreAllocate(PVM pVM);
2914	int pgmR3PhysRamReset(PVM pVM);
2915	int pgmR3PhysRomReset(PVM pVM);
2916	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
2917
2918	int pgmR3PoolInit(PVM pVM);
2919	void pgmR3PoolRelocate(PVM pVM);
2920	void pgmR3PoolReset(PVM pVM);
2921
2922	#endif /* IN_RING3 */
2923	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2924	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
2925	#endif
2926	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage);
2927	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
2928	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
2929	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2930	void pgmPoolFlushAll(PVM pVM);
2931	void pgmPoolClearAll(PVM pVM);
2932	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
2933	int pgmPoolSyncCR3(PVM pVM);
2934	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs);
2935	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT);
2936	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage);
2937	#ifdef PGMPOOL_WITH_MONITORING
2938	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu);
2939	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2940	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2941	#endif
2942
2943	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
2944	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
2945
2946	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
2947	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
2948	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2949	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2950
2951	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2952	#ifndef IN_RC
2953	int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2954	#endif
2955	int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
2956
2957	PX86PD pgmGstLazyMap32BitPD(PPGMCPU pPGM);
2958	PX86PDPT pgmGstLazyMapPaePDPT(PPGMCPU pPGM);
2959	PX86PDPAE pgmGstLazyMapPaePD(PPGMCPU pPGM, uint32_t iPdpt);
2960	PX86PML4 pgmGstLazyMapPml4(PPGMCPU pPGM);
2961
2962	__END_DECLS
2963
2964
2965	/**
2966	* Gets the PGMRAMRANGE structure for a guest page.
2967	*
2968	* @returns Pointer to the RAM range on success.
2969	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
2970	*
2971	* @param pPGM PGM handle.
2972	* @param GCPhys The GC physical address.
2973	*/
2974	DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
2975	{
2976	/*
2977	* Optimize for the first range.
2978	*/
2979	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
2980	RTGCPHYS off = GCPhys - pRam->GCPhys;
2981	if (RT_UNLIKELY(off >= pRam->cb))
2982	{
2983	do
2984	{
2985	pRam = pRam->CTX_SUFF(pNext);
2986	if (RT_UNLIKELY(!pRam))
2987	break;
2988	off = GCPhys - pRam->GCPhys;
2989	} while (off >= pRam->cb);
2990	}
2991	return pRam;
2992	}
2993
2994
2995	/**
2996	* Gets the PGMPAGE structure for a guest page.
2997	*
2998	* @returns Pointer to the page on success.
2999	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3000	*
3001	* @param pPGM PGM handle.
3002	* @param GCPhys The GC physical address.
3003	*/
3004	DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
3005	{
3006	/*
3007	* Optimize for the first range.
3008	*/
3009	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3010	RTGCPHYS off = GCPhys - pRam->GCPhys;
3011	if (RT_UNLIKELY(off >= pRam->cb))
3012	{
3013	do
3014	{
3015	pRam = pRam->CTX_SUFF(pNext);
3016	if (RT_UNLIKELY(!pRam))
3017	return NULL;
3018	off = GCPhys - pRam->GCPhys;
3019	} while (off >= pRam->cb);
3020	}
3021	return &pRam->aPages[off >> PAGE_SHIFT];
3022	}
3023
3024
3025	/**
3026	* Gets the PGMPAGE structure for a guest page.
3027	*
3028	* Old Phys code: Will make sure the page is present.
3029	*
3030	* @returns VBox status code.
3031	* @retval VINF_SUCCESS and a valid *ppPage on success.
3032	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3033	*
3034	* @param pPGM PGM handle.
3035	* @param GCPhys The GC physical address.
3036	* @param ppPage Where to store the page poitner on success.
3037	*/
3038	DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
3039	{
3040	/*
3041	* Optimize for the first range.
3042	*/
3043	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3044	RTGCPHYS off = GCPhys - pRam->GCPhys;
3045	if (RT_UNLIKELY(off >= pRam->cb))
3046	{
3047	do
3048	{
3049	pRam = pRam->CTX_SUFF(pNext);
3050	if (RT_UNLIKELY(!pRam))
3051	{
3052	ppPage = NULL; / avoid incorrect and very annoying GCC warnings */
3053	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3054	}
3055	off = GCPhys - pRam->GCPhys;
3056	} while (off >= pRam->cb);
3057	}
3058	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3059	return VINF_SUCCESS;
3060	}
3061
3062
3063
3064
3065	/**
3066	* Gets the PGMPAGE structure for a guest page.
3067	*
3068	* Old Phys code: Will make sure the page is present.
3069	*
3070	* @returns VBox status code.
3071	* @retval VINF_SUCCESS and a valid *ppPage on success.
3072	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3073	*
3074	* @param pPGM PGM handle.
3075	* @param GCPhys The GC physical address.
3076	* @param ppPage Where to store the page poitner on success.
3077	* @param ppRamHint Where to read and store the ram list hint.
3078	* The caller initializes this to NULL before the call.
3079	*/
3080	DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
3081	{
3082	RTGCPHYS off;
3083	PPGMRAMRANGE pRam = *ppRamHint;
3084	if ( !pRam
3085	\|\| RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
3086	{
3087	pRam = pPGM->CTX_SUFF(pRamRanges);
3088	off = GCPhys - pRam->GCPhys;
3089	if (RT_UNLIKELY(off >= pRam->cb))
3090	{
3091	do
3092	{
3093	pRam = pRam->CTX_SUFF(pNext);
3094	if (RT_UNLIKELY(!pRam))
3095	{
3096	ppPage = NULL; / Kill the incorrect and extremely annoying GCC warnings. */
3097	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3098	}
3099	off = GCPhys - pRam->GCPhys;
3100	} while (off >= pRam->cb);
3101	}
3102	*ppRamHint = pRam;
3103	}
3104	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3105	return VINF_SUCCESS;
3106	}
3107
3108
3109	/**
3110	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3111	*
3112	* @returns Pointer to the page on success.
3113	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3114	*
3115	* @param pPGM PGM handle.
3116	* @param GCPhys The GC physical address.
3117	* @param ppRam Where to store the pointer to the PGMRAMRANGE.
3118	*/
3119	DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
3120	{
3121	/*
3122	* Optimize for the first range.
3123	*/
3124	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3125	RTGCPHYS off = GCPhys - pRam->GCPhys;
3126	if (RT_UNLIKELY(off >= pRam->cb))
3127	{
3128	do
3129	{
3130	pRam = pRam->CTX_SUFF(pNext);
3131	if (RT_UNLIKELY(!pRam))
3132	return NULL;
3133	off = GCPhys - pRam->GCPhys;
3134	} while (off >= pRam->cb);
3135	}
3136	*ppRam = pRam;
3137	return &pRam->aPages[off >> PAGE_SHIFT];
3138	}
3139
3140
3141	/**
3142	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3143	*
3144	* @returns Pointer to the page on success.
3145	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3146	*
3147	* @param pPGM PGM handle.
3148	* @param GCPhys The GC physical address.
3149	* @param ppPage Where to store the pointer to the PGMPAGE structure.
3150	* @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
3151	*/
3152	DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
3153	{
3154	/*
3155	* Optimize for the first range.
3156	*/
3157	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3158	RTGCPHYS off = GCPhys - pRam->GCPhys;
3159	if (RT_UNLIKELY(off >= pRam->cb))
3160	{
3161	do
3162	{
3163	pRam = pRam->CTX_SUFF(pNext);
3164	if (RT_UNLIKELY(!pRam))
3165	{
3166	ppRam = NULL; / Shut up silly GCC warnings. */
3167	ppPage = NULL; / ditto */
3168	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3169	}
3170	off = GCPhys - pRam->GCPhys;
3171	} while (off >= pRam->cb);
3172	}
3173	*ppRam = pRam;
3174	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3175	return VINF_SUCCESS;
3176	}
3177
3178
3179	/**
3180	* Convert GC Phys to HC Phys.
3181	*
3182	* @returns VBox status.
3183	* @param pPGM PGM handle.
3184	* @param GCPhys The GC physical address.
3185	* @param pHCPhys Where to store the corresponding HC physical address.
3186	*
3187	* @deprecated Doesn't deal with zero, shared or write monitored pages.
3188	* Avoid when writing new code!
3189	*/
3190	DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
3191	{
3192	PPGMPAGE pPage;
3193	int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
3194	if (RT_FAILURE(rc))
3195	return rc;
3196	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
3197	return VINF_SUCCESS;
3198	}
3199
3200	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3201
3202	/**
3203	* Inlined version of the ring-0 version of PGMDynMapHCPage that
3204	* optimizes access to pages already in the set.
3205	*
3206	* @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
3207	* @param pPGM Pointer to the PVM instance data.
3208	* @param HCPhys The physical address of the page.
3209	* @param ppv Where to store the mapping address.
3210	*/
3211	DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
3212	{
3213	PVM pVM = PGM2VM(pPGM);
3214	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3215	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3216
3217	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
3218	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3219	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3220
3221	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3222	unsigned iEntry = pSet->aiHashTable[iHash];
3223	if ( iEntry < pSet->cEntries
3224	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3225	{
3226	*ppv = pSet->aEntries[iEntry].pvPage;
3227	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
3228	}
3229	else
3230	{
3231	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
3232	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3233	}
3234
3235	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
3236	return VINF_SUCCESS;
3237	}
3238
3239
3240	/**
3241	* Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
3242	* access to pages already in the set.
3243	*
3244	* @returns See PGMDynMapGCPage.
3245	* @param pPGM Pointer to the PVM instance data.
3246	* @param HCPhys The physical address of the page.
3247	* @param ppv Where to store the mapping address.
3248	*/
3249	DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3250	{
3251	PVM pVM = PGM2VM(pPGM);
3252	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3253
3254	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3255	Assert(!(GCPhys & PAGE_OFFSET_MASK));
3256
3257	/*
3258	* Get the ram range.
3259	*/
3260	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3261	RTGCPHYS off = GCPhys - pRam->GCPhys;
3262	if (RT_UNLIKELY(off >= pRam->cb
3263	/** @todo \|\| page state stuff */))
3264	{
3265	/* This case is not counted into StatR0DynMapGCPageInl. */
3266	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3267	return PGMDynMapGCPage(pVM, GCPhys, ppv);
3268	}
3269
3270	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3271	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3272
3273	/*
3274	* pgmR0DynMapHCPageInlined with out stats.
3275	*/
3276	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3277	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3278	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3279
3280	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3281	unsigned iEntry = pSet->aiHashTable[iHash];
3282	if ( iEntry < pSet->cEntries
3283	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3284	{
3285	*ppv = pSet->aEntries[iEntry].pvPage;
3286	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3287	}
3288	else
3289	{
3290	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3291	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3292	}
3293
3294	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3295	return VINF_SUCCESS;
3296	}
3297
3298	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3299	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3300
3301	/**
3302	* Maps the page into current context (RC and maybe R0).
3303	*
3304	* @returns pointer to the mapping.
3305	* @param pVM Pointer to the PGM instance data.
3306	* @param pPage The page.
3307	*/
3308	DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
3309	{
3310	if (pPage->idx >= PGMPOOL_IDX_FIRST)
3311	{
3312	Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
3313	void *pv;
3314	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3315	pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
3316	# else
3317	PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
3318	# endif
3319	return pv;
3320	}
3321	AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
3322	}
3323
3324	/**
3325	* Temporarily maps one host page specified by HC physical address, returning
3326	* pointer within the page.
3327	*
3328	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
3329	* reused after 8 mappings (or perhaps a few more if you score with the cache).
3330	*
3331	* @returns The address corresponding to HCPhys.
3332	* @param pPGM Pointer to the PVM instance data.
3333	* @param HCPhys HC Physical address of the page.
3334	*/
3335	DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
3336	{
3337	void *pv;
3338	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3339	pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3340	# else
3341	PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3342	# endif
3343	pv = (void *)((uintptr_t)pv \| (HCPhys & PAGE_OFFSET_MASK));
3344	return pv;
3345	}
3346
3347	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 \|\| IN_RC */
3348	#ifndef IN_RC
3349
3350	/**
3351	* Queries the Physical TLB entry for a physical guest page,
3352	* attempting to load the TLB entry if necessary.
3353	*
3354	* @returns VBox status code.
3355	* @retval VINF_SUCCESS on success
3356	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3357	*
3358	* @param pPGM The PGM instance handle.
3359	* @param GCPhys The address of the guest page.
3360	* @param ppTlbe Where to store the pointer to the TLB entry.
3361	*/
3362	DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3363	{
3364	int rc;
3365	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3366	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3367	{
3368	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3369	rc = VINF_SUCCESS;
3370	}
3371	else
3372	rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
3373	*ppTlbe = pTlbe;
3374	return rc;
3375	}
3376
3377
3378	/**
3379	* Queries the Physical TLB entry for a physical guest page,
3380	* attempting to load the TLB entry if necessary.
3381	*
3382	* @returns VBox status code.
3383	* @retval VINF_SUCCESS on success
3384	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3385	*
3386	* @param pPGM The PGM instance handle.
3387	* @param pPage Pointer to the PGMPAGE structure corresponding to
3388	* GCPhys.
3389	* @param GCPhys The address of the guest page.
3390	* @param ppTlbe Where to store the pointer to the TLB entry.
3391	*/
3392	DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3393	{
3394	int rc;
3395	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3396	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3397	{
3398	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3399	rc = VINF_SUCCESS;
3400	}
3401	else
3402	rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
3403	*ppTlbe = pTlbe;
3404	return rc;
3405	}
3406
3407	#endif /* !IN_RC */
3408
3409	/**
3410	* Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
3411	* Takes PSE-36 into account.
3412	*
3413	* @returns guest physical address
3414	* @param pPGM Pointer to the PGM instance data.
3415	* @param Pde Guest Pde
3416	*/
3417	DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
3418	{
3419	RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
3420	GCPhys \|= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
3421
3422	return GCPhys & pPGM->GCPhys4MBPSEMask;
3423	}
3424
3425
3426	/**
3427	* Gets the page directory entry for the specified address (32-bit paging).
3428	*
3429	* @returns The page directory entry in question.
3430	* @param pPGM Pointer to the PGM instance data.
3431	* @param GCPtr The address.
3432	*/
3433	DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3434	{
3435	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3436	PCX86PD pGuestPD = NULL;
3437	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3438	if (RT_FAILURE(rc))
3439	{
3440	X86PDE ZeroPde = {0};
3441	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
3442	}
3443	#else
3444	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3445	# ifdef IN_RING3
3446	if (!pGuestPD)
3447	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3448	# endif
3449	#endif
3450	return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3451	}
3452
3453
3454	/**
3455	* Gets the address of a specific page directory entry (32-bit paging).
3456	*
3457	* @returns Pointer the page directory entry in question.
3458	* @param pPGM Pointer to the PGM instance data.
3459	* @param GCPtr The address.
3460	*/
3461	DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3462	{
3463	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3464	PX86PD pGuestPD = NULL;
3465	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3466	AssertRCReturn(rc, NULL);
3467	#else
3468	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3469	# ifdef IN_RING3
3470	if (!pGuestPD)
3471	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3472	# endif
3473	#endif
3474	return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3475	}
3476
3477
3478	/**
3479	* Gets the address the guest page directory (32-bit paging).
3480	*
3481	* @returns Pointer the page directory entry in question.
3482	* @param pPGM Pointer to the PGM instance data.
3483	*/
3484	DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
3485	{
3486	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3487	PX86PD pGuestPD = NULL;
3488	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3489	AssertRCReturn(rc, NULL);
3490	#else
3491	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3492	# ifdef IN_RING3
3493	if (!pGuestPD)
3494	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3495	# endif
3496	#endif
3497	return pGuestPD;
3498	}
3499
3500
3501	/**
3502	* Gets the guest page directory pointer table.
3503	*
3504	* @returns Pointer to the page directory in question.
3505	* @returns NULL if the page directory is not present or on an invalid page.
3506	* @param pPGM Pointer to the PGM instance data.
3507	*/
3508	DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
3509	{
3510	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3511	PX86PDPT pGuestPDPT = NULL;
3512	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3513	AssertRCReturn(rc, NULL);
3514	#else
3515	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3516	# ifdef IN_RING3
3517	if (!pGuestPDPT)
3518	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3519	# endif
3520	#endif
3521	return pGuestPDPT;
3522	}
3523
3524
3525	/**
3526	* Gets the guest page directory pointer table entry for the specified address.
3527	*
3528	* @returns Pointer to the page directory in question.
3529	* @returns NULL if the page directory is not present or on an invalid page.
3530	* @param pPGM Pointer to the PGM instance data.
3531	* @param GCPtr The address.
3532	*/
3533	DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3534	{
3535	AssertGCPtr32(GCPtr);
3536
3537	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3538	PX86PDPT pGuestPDPT = 0;
3539	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3540	AssertRCReturn(rc, 0);
3541	#else
3542	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3543	# ifdef IN_RING3
3544	if (!pGuestPDPT)
3545	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3546	# endif
3547	#endif
3548	return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
3549	}
3550
3551
3552	/**
3553	* Gets the page directory for the specified address.
3554	*
3555	* @returns Pointer to the page directory in question.
3556	* @returns NULL if the page directory is not present or on an invalid page.
3557	* @param pPGM Pointer to the PGM instance data.
3558	* @param GCPtr The address.
3559	*/
3560	DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
3561	{
3562	AssertGCPtr32(GCPtr);
3563
3564	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3565	AssertReturn(pGuestPDPT, NULL);
3566	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3567	if (pGuestPDPT->a[iPdpt].n.u1Present)
3568	{
3569	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3570	PX86PDPAE pGuestPD = NULL;
3571	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3572	AssertRCReturn(rc, NULL);
3573	#else
3574	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3575	if ( !pGuestPD
3576	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3577	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3578	#endif
3579	return pGuestPD;
3580	/* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
3581	}
3582	return NULL;
3583	}
3584
3585
3586	/**
3587	* Gets the page directory entry for the specified address.
3588	*
3589	* @returns Pointer to the page directory entry in question.
3590	* @returns NULL if the page directory is not present or on an invalid page.
3591	* @param pPGM Pointer to the PGM instance data.
3592	* @param GCPtr The address.
3593	*/
3594	DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3595	{
3596	AssertGCPtr32(GCPtr);
3597
3598	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3599	AssertReturn(pGuestPDPT, NULL);
3600	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3601	if (pGuestPDPT->a[iPdpt].n.u1Present)
3602	{
3603	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3604	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3605	PX86PDPAE pGuestPD = NULL;
3606	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3607	AssertRCReturn(rc, NULL);
3608	#else
3609	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3610	if ( !pGuestPD
3611	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3612	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3613	#endif
3614	return &pGuestPD->a[iPD];
3615	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
3616	}
3617	return NULL;
3618	}
3619
3620
3621	/**
3622	* Gets the page directory entry for the specified address.
3623	*
3624	* @returns The page directory entry in question.
3625	* @returns A non-present entry if the page directory is not present or on an invalid page.
3626	* @param pPGM Pointer to the PGM instance data.
3627	* @param GCPtr The address.
3628	*/
3629	DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3630	{
3631	AssertGCPtr32(GCPtr);
3632	X86PDEPAE ZeroPde = {0};
3633	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3634	if (RT_LIKELY(pGuestPDPT))
3635	{
3636	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3637	if (pGuestPDPT->a[iPdpt].n.u1Present)
3638	{
3639	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3640	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3641	PX86PDPAE pGuestPD = NULL;
3642	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3643	AssertRCReturn(rc, ZeroPde);
3644	#else
3645	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3646	if ( !pGuestPD
3647	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3648	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3649	#endif
3650	return pGuestPD->a[iPD];
3651	}
3652	}
3653	return ZeroPde;
3654	}
3655
3656
3657	/**
3658	* Gets the page directory pointer table entry for the specified address
3659	* and returns the index into the page directory
3660	*
3661	* @returns Pointer to the page directory in question.
3662	* @returns NULL if the page directory is not present or on an invalid page.
3663	* @param pPGM Pointer to the PGM instance data.
3664	* @param GCPtr The address.
3665	* @param piPD Receives the index into the returned page directory
3666	* @param pPdpe Receives the page directory pointer entry. Optional.
3667	*/
3668	DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
3669	{
3670	AssertGCPtr32(GCPtr);
3671
3672	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3673	AssertReturn(pGuestPDPT, NULL);
3674	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3675	if (pPdpe)
3676	*pPdpe = pGuestPDPT->a[iPdpt];
3677	if (pGuestPDPT->a[iPdpt].n.u1Present)
3678	{
3679	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3680	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3681	PX86PDPAE pGuestPD = NULL;
3682	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3683	AssertRCReturn(rc, NULL);
3684	#else
3685	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3686	if ( !pGuestPD
3687	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3688	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3689	#endif
3690	*piPD = iPD;
3691	return pGuestPD;
3692	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
3693	}
3694	return NULL;
3695	}
3696
3697	#ifndef IN_RC
3698
3699	/**
3700	* Gets the page map level-4 pointer for the guest.
3701	*
3702	* @returns Pointer to the PML4 page.
3703	* @param pPGM Pointer to the PGM instance data.
3704	*/
3705	DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
3706	{
3707	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3708	PX86PML4 pGuestPml4;
3709	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3710	AssertRCReturn(rc, NULL);
3711	#else
3712	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3713	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3714	if (!pGuestPml4)
3715	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3716	# endif
3717	Assert(pGuestPml4);
3718	#endif
3719	return pGuestPml4;
3720	}
3721
3722
3723	/**
3724	* Gets the pointer to a page map level-4 entry.
3725	*
3726	* @returns Pointer to the PML4 entry.
3727	* @param pPGM Pointer to the PGM instance data.
3728	* @param iPml4 The index.
3729	*/
3730	DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
3731	{
3732	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3733	PX86PML4 pGuestPml4;
3734	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3735	AssertRCReturn(rc, NULL);
3736	#else
3737	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3738	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3739	if (!pGuestPml4)
3740	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3741	# endif
3742	Assert(pGuestPml4);
3743	#endif
3744	return &pGuestPml4->a[iPml4];
3745	}
3746
3747
3748	/**
3749	* Gets a page map level-4 entry.
3750	*
3751	* @returns The PML4 entry.
3752	* @param pPGM Pointer to the PGM instance data.
3753	* @param iPml4 The index.
3754	*/
3755	DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
3756	{
3757	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3758	PX86PML4 pGuestPml4;
3759	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3760	if (RT_FAILURE(rc))
3761	{
3762	X86PML4E ZeroPml4e = {0};
3763	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
3764	}
3765	#else
3766	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3767	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3768	if (!pGuestPml4)
3769	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3770	# endif
3771	Assert(pGuestPml4);
3772	#endif
3773	return pGuestPml4->a[iPml4];
3774	}
3775
3776
3777	/**
3778	* Gets the page directory pointer entry for the specified address.
3779	*
3780	* @returns Pointer to the page directory pointer entry in question.
3781	* @returns NULL if the page directory is not present or on an invalid page.
3782	* @param pPGM Pointer to the PGM instance data.
3783	* @param GCPtr The address.
3784	* @param ppPml4e Page Map Level-4 Entry (out)
3785	*/
3786	DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
3787	{
3788	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3789	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3790	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3791	if (pPml4e->n.u1Present)
3792	{
3793	PX86PDPT pPdpt;
3794	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
3795	AssertRCReturn(rc, NULL);
3796
3797	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3798	return &pPdpt->a[iPdpt];
3799	}
3800	return NULL;
3801	}
3802
3803
3804	/**
3805	* Gets the page directory entry for the specified address.
3806	*
3807	* @returns The page directory entry in question.
3808	* @returns A non-present entry if the page directory is not present or on an invalid page.
3809	* @param pPGM Pointer to the PGM instance data.
3810	* @param GCPtr The address.
3811	* @param ppPml4e Page Map Level-4 Entry (out)
3812	* @param pPdpe Page directory pointer table entry (out)
3813	*/
3814	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
3815	{
3816	X86PDEPAE ZeroPde = {0};
3817	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3818	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3819	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3820	if (pPml4e->n.u1Present)
3821	{
3822	PCX86PDPT pPdptTemp;
3823	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3824	AssertRCReturn(rc, ZeroPde);
3825
3826	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3827	*pPdpe = pPdptTemp->a[iPdpt];
3828	if (pPdptTemp->a[iPdpt].n.u1Present)
3829	{
3830	PCX86PDPAE pPD;
3831	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3832	AssertRCReturn(rc, ZeroPde);
3833
3834	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3835	return pPD->a[iPD];
3836	}
3837	}
3838
3839	return ZeroPde;
3840	}
3841
3842
3843	/**
3844	* Gets the page directory entry for the specified address.
3845	*
3846	* @returns The page directory entry in question.
3847	* @returns A non-present entry if the page directory is not present or on an invalid page.
3848	* @param pPGM Pointer to the PGM instance data.
3849	* @param GCPtr The address.
3850	*/
3851	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3852	{
3853	X86PDEPAE ZeroPde = {0};
3854	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3855	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3856	if (pGuestPml4->a[iPml4].n.u1Present)
3857	{
3858	PCX86PDPT pPdptTemp;
3859	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3860	AssertRCReturn(rc, ZeroPde);
3861
3862	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3863	if (pPdptTemp->a[iPdpt].n.u1Present)
3864	{
3865	PCX86PDPAE pPD;
3866	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3867	AssertRCReturn(rc, ZeroPde);
3868
3869	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3870	return pPD->a[iPD];
3871	}
3872	}
3873	return ZeroPde;
3874	}
3875
3876
3877	/**
3878	* Gets the page directory entry for the specified address.
3879	*
3880	* @returns Pointer to the page directory entry in question.
3881	* @returns NULL if the page directory is not present or on an invalid page.
3882	* @param pPGM Pointer to the PGM instance data.
3883	* @param GCPtr The address.
3884	*/
3885	DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3886	{
3887	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3888	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3889	if (pGuestPml4->a[iPml4].n.u1Present)
3890	{
3891	PCX86PDPT pPdptTemp;
3892	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3893	AssertRCReturn(rc, NULL);
3894
3895	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3896	if (pPdptTemp->a[iPdpt].n.u1Present)
3897	{
3898	PX86PDPAE pPD;
3899	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3900	AssertRCReturn(rc, NULL);
3901
3902	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3903	return &pPD->a[iPD];
3904	}
3905	}
3906	return NULL;
3907	}
3908
3909
3910	/**
3911	* Gets the GUEST page directory pointer for the specified address.
3912	*
3913	* @returns The page directory in question.
3914	* @returns NULL if the page directory is not present or on an invalid page.
3915	* @param pPGM Pointer to the PGM instance data.
3916	* @param GCPtr The address.
3917	* @param ppPml4e Page Map Level-4 Entry (out)
3918	* @param pPdpe Page directory pointer table entry (out)
3919	* @param piPD Receives the index into the returned page directory
3920	*/
3921	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPE pPdpe, unsigned piPD)
3922	{
3923	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3924	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3925	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3926	if (pPml4e->n.u1Present)
3927	{
3928	PCX86PDPT pPdptTemp;
3929	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3930	AssertRCReturn(rc, NULL);
3931
3932	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3933	*pPdpe = pPdptTemp->a[iPdpt];
3934	if (pPdptTemp->a[iPdpt].n.u1Present)
3935	{
3936	PX86PDPAE pPD;
3937	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3938	AssertRCReturn(rc, NULL);
3939
3940	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3941	return pPD;
3942	}
3943	}
3944	return 0;
3945	}
3946
3947	#endif /* !IN_RC */
3948
3949	/**
3950	* Gets the shadow page directory, 32-bit.
3951	*
3952	* @returns Pointer to the shadow 32-bit PD.
3953	* @param pPGM Pointer to the PGM instance data.
3954	*/
3955	DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
3956	{
3957	return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
3958	}
3959
3960
3961	/**
3962	* Gets the shadow page directory entry for the specified address, 32-bit.
3963	*
3964	* @returns Shadow 32-bit PDE.
3965	* @param pPGM Pointer to the PGM instance data.
3966	* @param GCPtr The address.
3967	*/
3968	DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3969	{
3970	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
3971
3972	PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
3973	if (!pShwPde)
3974	{
3975	X86PDE ZeroPde = {0};
3976	return ZeroPde;
3977	}
3978	return pShwPde->a[iPd];
3979	}
3980
3981
3982	/**
3983	* Gets the pointer to the shadow page directory entry for the specified
3984	* address, 32-bit.
3985	*
3986	* @returns Pointer to the shadow 32-bit PDE.
3987	* @param pPGM Pointer to the PGM instance data.
3988	* @param GCPtr The address.
3989	*/
3990	DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3991	{
3992	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
3993
3994	PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
3995	AssertReturn(pPde, NULL);
3996	return &pPde->a[iPd];
3997	}
3998
3999
4000	/**
4001	* Gets the shadow page pointer table, PAE.
4002	*
4003	* @returns Pointer to the shadow PAE PDPT.
4004	* @param pPGM Pointer to the PGM instance data.
4005	*/
4006	DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
4007	{
4008	return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4009	}
4010
4011
4012	/**
4013	* Gets the shadow page directory for the specified address, PAE.
4014	*
4015	* @returns Pointer to the shadow PD.
4016	* @param pPGM Pointer to the PGM instance data.
4017	* @param GCPtr The address.
4018	*/
4019	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4020	{
4021	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4022	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
4023
4024	if (!pPdpt->a[iPdpt].n.u1Present)
4025	return NULL;
4026
4027	/* Fetch the pgm pool shadow descriptor. */
4028	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4029	AssertReturn(pShwPde, NULL);
4030
4031	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4032	}
4033
4034
4035	/**
4036	* Gets the shadow page directory for the specified address, PAE.
4037	*
4038	* @returns Pointer to the shadow PD.
4039	* @param pPGM Pointer to the PGM instance data.
4040	* @param GCPtr The address.
4041	*/
4042	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
4043	{
4044	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4045
4046	if (!pPdpt->a[iPdpt].n.u1Present)
4047	return NULL;
4048
4049	/* Fetch the pgm pool shadow descriptor. */
4050	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4051	AssertReturn(pShwPde, NULL);
4052
4053	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4054	}
4055
4056
4057	/**
4058	* Gets the shadow page directory entry, PAE.
4059	*
4060	* @returns PDE.
4061	* @param pPGM Pointer to the PGM instance data.
4062	* @param GCPtr The address.
4063	*/
4064	DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4065	{
4066	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4067
4068	PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4069	if (!pShwPde)
4070	{
4071	X86PDEPAE ZeroPde = {0};
4072	return ZeroPde;
4073	}
4074	return pShwPde->a[iPd];
4075	}
4076
4077
4078	/**
4079	* Gets the pointer to the shadow page directory entry for an address, PAE.
4080	*
4081	* @returns Pointer to the PDE.
4082	* @param pPGM Pointer to the PGM instance data.
4083	* @param GCPtr The address.
4084	*/
4085	DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4086	{
4087	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4088
4089	PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4090	AssertReturn(pPde, NULL);
4091	return &pPde->a[iPd];
4092	}
4093
4094	#ifndef IN_RC
4095
4096	/**
4097	* Gets the shadow page map level-4 pointer.
4098	*
4099	* @returns Pointer to the shadow PML4.
4100	* @param pPGM Pointer to the PGM instance data.
4101	*/
4102	DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
4103	{
4104	return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4105	}
4106
4107
4108	/**
4109	* Gets the shadow page map level-4 entry for the specified address.
4110	*
4111	* @returns The entry.
4112	* @param pPGM Pointer to the PGM instance data.
4113	* @param GCPtr The address.
4114	*/
4115	DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
4116	{
4117	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4118	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4119
4120	if (!pShwPml4)
4121	{
4122	X86PML4E ZeroPml4e = {0};
4123	return ZeroPml4e;
4124	}
4125	return pShwPml4->a[iPml4];
4126	}
4127
4128
4129	/**
4130	* Gets the pointer to the specified shadow page map level-4 entry.
4131	*
4132	* @returns The entry.
4133	* @param pPGM Pointer to the PGM instance data.
4134	* @param iPml4 The PML4 index.
4135	*/
4136	DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4137	{
4138	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4139	if (!pShwPml4)
4140	return NULL;
4141	return &pShwPml4->a[iPml4];
4142	}
4143
4144
4145	/**
4146	* Gets the GUEST page directory pointer for the specified address.
4147	*
4148	* @returns The page directory in question.
4149	* @returns NULL if the page directory is not present or on an invalid page.
4150	* @param pPGM Pointer to the PGM instance data.
4151	* @param GCPtr The address.
4152	* @param piPD Receives the index into the returned page directory
4153	*/
4154	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
4155	{
4156	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4157	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4158	if (pGuestPml4->a[iPml4].n.u1Present)
4159	{
4160	PCX86PDPT pPdptTemp;
4161	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4162	AssertRCReturn(rc, NULL);
4163
4164	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4165	if (pPdptTemp->a[iPdpt].n.u1Present)
4166	{
4167	PX86PDPAE pPD;
4168	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4169	AssertRCReturn(rc, NULL);
4170
4171	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4172	return pPD;
4173	}
4174	}
4175	return NULL;
4176	}
4177
4178	#endif /* !IN_RC */
4179
4180	/**
4181	* Gets the page state for a physical handler.
4182	*
4183	* @returns The physical handler page state.
4184	* @param pCur The physical handler in question.
4185	*/
4186	DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
4187	{
4188	switch (pCur->enmType)
4189	{
4190	case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
4191	return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
4192
4193	case PGMPHYSHANDLERTYPE_MMIO:
4194	case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
4195	return PGM_PAGE_HNDL_PHYS_STATE_ALL;
4196
4197	default:
4198	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4199	}
4200	}
4201
4202
4203	/**
4204	* Gets the page state for a virtual handler.
4205	*
4206	* @returns The virtual handler page state.
4207	* @param pCur The virtual handler in question.
4208	* @remarks This should never be used on a hypervisor access handler.
4209	*/
4210	DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
4211	{
4212	switch (pCur->enmType)
4213	{
4214	case PGMVIRTHANDLERTYPE_WRITE:
4215	return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
4216	case PGMVIRTHANDLERTYPE_ALL:
4217	return PGM_PAGE_HNDL_VIRT_STATE_ALL;
4218	default:
4219	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4220	}
4221	}
4222
4223
4224	/**
4225	* Clears one physical page of a virtual handler
4226	*
4227	* @param pPGM Pointer to the PGM instance.
4228	* @param pCur Virtual handler structure
4229	* @param iPage Physical page index
4230	*
4231	* @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
4232	* need to care about other handlers in the same page.
4233	*/
4234	DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
4235	{
4236	const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
4237
4238	/*
4239	* Remove the node from the tree (it's supposed to be in the tree if we get here!).
4240	*/
4241	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4242	AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4243	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4244	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4245	#endif
4246	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
4247	{
4248	/* We're the head of the alias chain. */
4249	PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
4250	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4251	AssertReleaseMsg(pRemove != NULL,
4252	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4253	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4254	AssertReleaseMsg(pRemove == pPhys2Virt,
4255	("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4256	" got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4257	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
4258	pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
4259	#endif
4260	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4261	{
4262	/* Insert the next list in the alias chain into the tree. */
4263	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4264	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4265	AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4266	("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4267	pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
4268	#endif
4269	pNext->offNextAlias \|= PGMPHYS2VIRTHANDLER_IS_HEAD;
4270	bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
4271	AssertRelease(fRc);
4272	}
4273	}
4274	else
4275	{
4276	/* Locate the previous node in the alias chain. */
4277	PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
4278	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4279	AssertReleaseMsg(pPrev != pPhys2Virt,
4280	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4281	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4282	#endif
4283	for (;;)
4284	{
4285	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4286	if (pNext == pPhys2Virt)
4287	{
4288	/* unlink. */
4289	LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
4290	pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
4291	if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4292	pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
4293	else
4294	{
4295	PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4296	pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
4297	\| (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
4298	}
4299	break;
4300	}
4301
4302	/* next */
4303	if (pNext == pPrev)
4304	{
4305	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4306	AssertReleaseMsg(pNext != pPrev,
4307	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4308	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4309	#endif
4310	break;
4311	}
4312	pPrev = pNext;
4313	}
4314	}
4315	Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
4316	pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
4317	pPhys2Virt->offNextAlias = 0;
4318	pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
4319
4320	/*
4321	* Clear the ram flags for this page.
4322	*/
4323	PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
4324	AssertReturnVoid(pPage);
4325	PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
4326	}
4327
4328
4329	/**
4330	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4331	*
4332	* @returns Pointer to the shadow page structure.
4333	* @param pPool The pool.
4334	* @param idx The pool page index.
4335	*/
4336	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
4337	{
4338	AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
4339	return &pPool->aPages[idx];
4340	}
4341
4342
4343	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4344	/**
4345	* Clear references to guest physical memory.
4346	*
4347	* @param pPool The pool.
4348	* @param pPoolPage The pool page.
4349	* @param pPhysPage The physical guest page tracking structure.
4350	*/
4351	DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
4352	{
4353	/*
4354	* Just deal with the simple case here.
4355	*/
4356	# ifdef LOG_ENABLED
4357	const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
4358	# endif
4359	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4360	if (cRefs == 1)
4361	{
4362	Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
4363	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
4364	}
4365	else
4366	pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
4367	Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
4368	}
4369	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4370
4371
4372	#ifdef PGMPOOL_WITH_CACHE
4373	/**
4374	* Moves the page to the head of the age list.
4375	*
4376	* This is done when the cached page is used in one way or another.
4377	*
4378	* @param pPool The pool.
4379	* @param pPage The cached page.
4380	*/
4381	DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4382	{
4383	PVM pVM = pPool->CTX_SUFF(pVM);
4384	pgmLock(pVM);
4385
4386	/*
4387	* Move to the head of the age list.
4388	*/
4389	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
4390	{
4391	/* unlink */
4392	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
4393	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
4394	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
4395	else
4396	pPool->iAgeTail = pPage->iAgePrev;
4397
4398	/* insert at head */
4399	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4400	pPage->iAgeNext = pPool->iAgeHead;
4401	Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
4402	pPool->iAgeHead = pPage->idx;
4403	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
4404	}
4405	pgmUnlock(pVM);
4406	}
4407	#endif /* PGMPOOL_WITH_CACHE */
4408
4409	/**
4410	* Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
4411	*
4412	* @param pVM VM Handle.
4413	* @param pPage PGM pool page
4414	*/
4415	DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4416	{
4417	pPage->cLocked++;
4418	}
4419
4420
4421	/**
4422	* Unlocks a page to allow flushing again
4423	*
4424	* @param pVM VM Handle.
4425	* @param pPage PGM pool page
4426	*/
4427	DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4428	{
4429	Assert(pPage->cLocked);
4430	pPage->cLocked--;
4431	}
4432
4433
4434	/**
4435	* Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
4436	*
4437	* @returns VBox status code.
4438	* @param pPage PGM pool page
4439	*/
4440	DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
4441	{
4442	if (pPage->cLocked)
4443	{
4444	LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
4445	if (pPage->cModifications)
4446	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
4447	return true;
4448	}
4449	return false;
4450	}
4451
4452	/**
4453	* Tells if mappings are to be put into the shadow page table or not
4454	*
4455	* @returns boolean result
4456	* @param pVM VM handle.
4457	*/
4458	DECLINLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
4459	{
4460	#ifdef IN_RING0
4461	/* There are no mappings in VT-x and AMD-V mode. */
4462	Assert(pPGM->fDisableMappings);
4463	return false;
4464	#else
4465	return !pPGM->fDisableMappings;
4466	#endif
4467	}
4468
4469	/** @} */
4470
4471	#endif
4472
4473

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMInternal.h@ 19834

Download in other formats: