PGMInternal.h@ 19167

Last change on this file since 19167 was 19022, checked in by vboxsync, 16 years ago
MM,PGM: RamPreAlloc docs to PGM, MM doesn't need to query it any longer.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 185.8 KB

Line
1	/* $Id: PGMInternal.h 19022 2009-04-20 13:11:16Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	#ifndef ___PGMInternal_h
23	#define ___PGMInternal_h
24
25	#include <VBox/cdefs.h>
26	#include <VBox/types.h>
27	#include <VBox/err.h>
28	#include <VBox/stam.h>
29	#include <VBox/param.h>
30	#include <VBox/vmm.h>
31	#include <VBox/mm.h>
32	#include <VBox/pdmcritsect.h>
33	#include <VBox/pdmapi.h>
34	#include <VBox/dis.h>
35	#include <VBox/dbgf.h>
36	#include <VBox/log.h>
37	#include <VBox/gmm.h>
38	#include <VBox/hwaccm.h>
39	#include <iprt/avl.h>
40	#include <iprt/assert.h>
41	#include <iprt/critsect.h>
42
43
44
45	/** @defgroup grp_pgm_int Internals
46	* @ingroup grp_pgm
47	* @internal
48	* @{
49	*/
50
51
52	/** @name PGM Compile Time Config
53	* @{
54	*/
55
56	/**
57	* Solve page is out of sync issues inside Guest Context (in PGMGC.cpp).
58	* Comment it if it will break something.
59	*/
60	#define PGM_OUT_OF_SYNC_IN_GC
61
62	/**
63	* Check and skip global PDEs for non-global flushes
64	*/
65	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
66
67	/**
68	* Sync N pages instead of a whole page table
69	*/
70	#define PGM_SYNC_N_PAGES
71
72	/**
73	* Number of pages to sync during a page fault
74	*
75	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
76	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
77	*/
78	#define PGM_SYNC_NR_PAGES 8
79
80	/**
81	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
82	*/
83	#define PGM_MAX_PHYSCACHE_ENTRIES 64
84	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
85
86	/**
87	* Enable caching of PGMR3PhysRead/WriteByte/Word/Dword
88	*/
89	#define PGM_PHYSMEMACCESS_CACHING
90
91	/** @def PGMPOOL_WITH_CACHE
92	* Enable agressive caching using the page pool.
93	*
94	* This requires PGMPOOL_WITH_USER_TRACKING and PGMPOOL_WITH_MONITORING.
95	*/
96	#define PGMPOOL_WITH_CACHE
97
98	/** @def PGMPOOL_WITH_MIXED_PT_CR3
99	* When defined, we'll deal with 'uncachable' pages.
100	*/
101	#ifdef PGMPOOL_WITH_CACHE
102	# define PGMPOOL_WITH_MIXED_PT_CR3
103	#endif
104
105	/** @def PGMPOOL_WITH_MONITORING
106	* Monitor the guest pages which are shadowed.
107	* When this is enabled, PGMPOOL_WITH_CACHE or PGMPOOL_WITH_GCPHYS_TRACKING must
108	* be enabled as well.
109	* @remark doesn't really work without caching now. (Mixed PT/CR3 change.)
110	*/
111	#ifdef PGMPOOL_WITH_CACHE
112	# define PGMPOOL_WITH_MONITORING
113	#endif
114
115	/** @def PGMPOOL_WITH_GCPHYS_TRACKING
116	* Tracking the of shadow pages mapping guest physical pages.
117	*
118	* This is very expensive, the current cache prototype is trying to figure out
119	* whether it will be acceptable with an agressive caching policy.
120	*/
121	#if defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
122	# define PGMPOOL_WITH_GCPHYS_TRACKING
123	#endif
124
125	/** @def PGMPOOL_WITH_USER_TRACKING
126	* Tracking users of shadow pages. This is required for the linking of shadow page
127	* tables and physical guest addresses.
128	*/
129	#if defined(PGMPOOL_WITH_GCPHYS_TRACKING) \|\| defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
130	# define PGMPOOL_WITH_USER_TRACKING
131	#endif
132
133	/** @def PGMPOOL_CFG_MAX_GROW
134	* The maximum number of pages to add to the pool in one go.
135	*/
136	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
137
138	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
139	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
140	*/
141	#ifdef VBOX_STRICT
142	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
143	#endif
144
145	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
146	* Enables the experimental lazy page allocation code. */
147	/# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
148
149	/** @} */
150
151
152	/** @name PDPT and PML4 flags.
153	* These are placed in the three bits available for system programs in
154	* the PDPT and PML4 entries.
155	* @{ */
156	/** The entry is a permanent one and it's must always be present.
157	* Never free such an entry. */
158	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
159	/** Mapping (hypervisor allocated pagetable). */
160	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
161	/** @} */
162
163	/** @name Page directory flags.
164	* These are placed in the three bits available for system programs in
165	* the page directory entries.
166	* @{ */
167	/** Mapping (hypervisor allocated pagetable). */
168	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
169	/** Made read-only to facilitate dirty bit tracking. */
170	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
171	/** @} */
172
173	/** @name Page flags.
174	* These are placed in the three bits available for system programs in
175	* the page entries.
176	* @{ */
177	/** Made read-only to facilitate dirty bit tracking. */
178	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
179
180	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
181	/** Scanned and approved by CSAM (tm).
182	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
183	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
184	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
185	#endif
186
187	/** @} */
188
189	/** @name Defines used to indicate the shadow and guest paging in the templates.
190	* @{ */
191	#define PGM_TYPE_REAL 1
192	#define PGM_TYPE_PROT 2
193	#define PGM_TYPE_32BIT 3
194	#define PGM_TYPE_PAE 4
195	#define PGM_TYPE_AMD64 5
196	#define PGM_TYPE_NESTED 6
197	#define PGM_TYPE_EPT 7
198	#define PGM_TYPE_MAX PGM_TYPE_EPT
199	/** @} */
200
201	/** Macro for checking if the guest is using paging.
202	* @param uGstType PGM_TYPE_*
203	* @param uShwType PGM_TYPE_*
204	* @remark ASSUMES certain order of the PGM_TYPE_* values.
205	*/
206	#define PGM_WITH_PAGING(uGstType, uShwType) \
207	( (uGstType) >= PGM_TYPE_32BIT \
208	&& (uShwType) != PGM_TYPE_NESTED \
209	&& (uShwType) != PGM_TYPE_EPT)
210
211	/** Macro for checking if the guest supports the NX bit.
212	* @param uGstType PGM_TYPE_*
213	* @param uShwType PGM_TYPE_*
214	* @remark ASSUMES certain order of the PGM_TYPE_* values.
215	*/
216	#define PGM_WITH_NX(uGstType, uShwType) \
217	( (uGstType) >= PGM_TYPE_PAE \
218	&& (uShwType) != PGM_TYPE_NESTED \
219	&& (uShwType) != PGM_TYPE_EPT)
220
221
222	/** @def PGM_HCPHYS_2_PTR
223	* Maps a HC physical page pool address to a virtual address.
224	*
225	* @returns VBox status code.
226	* @param pVM The VM handle.
227	* @param HCPhys The HC physical address to map to a virtual one.
228	* @param ppv Where to store the virtual address. No need to cast this.
229	*
230	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
231	* small page window employeed by that function. Be careful.
232	* @remark There is no need to assert on the result.
233	*/
234	#ifdef IN_RC
235	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
236	PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
237	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
238	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
239	pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
240	#else
241	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
242	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
243	#endif
244
245	/** @def PGM_HCPHYS_2_PTR_BY_PGM
246	* Maps a HC physical page pool address to a virtual address.
247	*
248	* @returns VBox status code.
249	* @param pPGM The PGM instance data.
250	* @param HCPhys The HC physical address to map to a virtual one.
251	* @param ppv Where to store the virtual address. No need to cast this.
252	*
253	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
254	* small page window employeed by that function. Be careful.
255	* @remark There is no need to assert on the result.
256	*/
257	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
258	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
259	pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
260	#else
261	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
262	PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
263	#endif
264
265	/** @def PGM_GCPHYS_2_PTR
266	* Maps a GC physical page address to a virtual address.
267	*
268	* @returns VBox status code.
269	* @param pVM The VM handle.
270	* @param GCPhys The GC physical address to map to a virtual one.
271	* @param ppv Where to store the virtual address. No need to cast this.
272	*
273	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
274	* small page window employeed by that function. Be careful.
275	* @remark There is no need to assert on the result.
276	*/
277	#ifdef IN_RC
278	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
279	PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
280	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
281	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
282	pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
283	#else
284	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
285	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
286	#endif
287
288	/** @def PGM_GCPHYS_2_PTR_BY_PGMCPU
289	* Maps a GC physical page address to a virtual address.
290	*
291	* @returns VBox status code.
292	* @param pPGM Pointer to the PGM instance data.
293	* @param GCPhys The GC physical address to map to a virtual one.
294	* @param ppv Where to store the virtual address. No need to cast this.
295	*
296	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
297	* small page window employeed by that function. Be careful.
298	* @remark There is no need to assert on the result.
299	*/
300	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
301	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
302	pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), GCPhys, (void **)(ppv))
303	#else
304	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
305	PGM_GCPHYS_2_PTR(PGMCPU2VM(pPGM), GCPhys, ppv)
306	#endif
307
308	/** @def PGM_GCPHYS_2_PTR_EX
309	* Maps a unaligned GC physical page address to a virtual address.
310	*
311	* @returns VBox status code.
312	* @param pVM The VM handle.
313	* @param GCPhys The GC physical address to map to a virtual one.
314	* @param ppv Where to store the virtual address. No need to cast this.
315	*
316	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
317	* small page window employeed by that function. Be careful.
318	* @remark There is no need to assert on the result.
319	*/
320	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
321	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
322	PGMDynMapGCPageOff(pVM, GCPhys, (void **)(ppv))
323	#else
324	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
325	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
326	#endif
327
328	/** @def PGM_INVL_PG
329	* Invalidates a page when in GC does nothing in HC.
330	*
331	* @param GCVirt The virtual address of the page to invalidate.
332	*/
333	#ifdef IN_RC
334	# define PGM_INVL_PG(GCVirt) ASMInvalidatePage((void *)(GCVirt))
335	#elif defined(IN_RING0)
336	# define PGM_INVL_PG(GCVirt) HWACCMInvalidatePage(pVM, (RTGCPTR)(GCVirt))
337	#else
338	# define PGM_INVL_PG(GCVirt) HWACCMInvalidatePage(pVM, (RTGCPTR)(GCVirt))
339	#endif
340
341	/** @def PGM_INVL_BIG_PG
342	* Invalidates a 4MB page directory entry when in GC does nothing in HC.
343	*
344	* @param GCVirt The virtual address within the page directory to invalidate.
345	*/
346	#ifdef IN_RC
347	# define PGM_INVL_BIG_PG(GCVirt) ASMReloadCR3()
348	#elif defined(IN_RING0)
349	# define PGM_INVL_BIG_PG(GCVirt) HWACCMFlushTLB(pVM)
350	#else
351	# define PGM_INVL_BIG_PG(GCVirt) HWACCMFlushTLB(pVM)
352	#endif
353
354	/** @def PGM_INVL_GUEST_TLBS()
355	* Invalidates all guest TLBs.
356	*/
357	#ifdef IN_RC
358	# define PGM_INVL_GUEST_TLBS() ASMReloadCR3()
359	#elif defined(IN_RING0)
360	# define PGM_INVL_GUEST_TLBS() HWACCMFlushTLB(pVM)
361	#else
362	# define PGM_INVL_GUEST_TLBS() HWACCMFlushTLB(pVM)
363	#endif
364
365	/** Size of the GCPtrConflict array in PGMMAPPING.
366	* @remarks Must be a power of two. */
367	#define PGMMAPPING_CONFLICT_MAX 8
368
369	/**
370	* Structure for tracking GC Mappings.
371	*
372	* This structure is used by linked list in both GC and HC.
373	*/
374	typedef struct PGMMAPPING
375	{
376	/** Pointer to next entry. */
377	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
378	/** Pointer to next entry. */
379	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
380	/** Pointer to next entry. */
381	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
382	/** Indicate whether this entry is finalized. */
383	bool fFinalized;
384	/** Start Virtual address. */
385	RTGCPTR GCPtr;
386	/** Last Virtual address (inclusive). */
387	RTGCPTR GCPtrLast;
388	/** Range size (bytes). */
389	RTGCPTR cb;
390	/** Pointer to relocation callback function. */
391	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
392	/** User argument to the callback. */
393	R3PTRTYPE(void *) pvUser;
394	/** Mapping description / name. For easing debugging. */
395	R3PTRTYPE(const char *) pszDesc;
396	/** Last 8 addresses that caused conflicts. */
397	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
398	/** Number of conflicts for this hypervisor mapping. */
399	uint32_t cConflicts;
400	/** Number of page tables. */
401	uint32_t cPTs;
402
403	/** Array of page table mapping data. Each entry
404	* describes one page table. The array can be longer
405	* than the declared length.
406	*/
407	struct
408	{
409	/** The HC physical address of the page table. */
410	RTHCPHYS HCPhysPT;
411	/** The HC physical address of the first PAE page table. */
412	RTHCPHYS HCPhysPaePT0;
413	/** The HC physical address of the second PAE page table. */
414	RTHCPHYS HCPhysPaePT1;
415	/** The HC virtual address of the 32-bit page table. */
416	R3PTRTYPE(PX86PT) pPTR3;
417	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
418	R3PTRTYPE(PX86PTPAE) paPaePTsR3;
419	/** The GC virtual address of the 32-bit page table. */
420	RCPTRTYPE(PX86PT) pPTRC;
421	/** The GC virtual address of the two PAE page table. */
422	RCPTRTYPE(PX86PTPAE) paPaePTsRC;
423	/** The GC virtual address of the 32-bit page table. */
424	R0PTRTYPE(PX86PT) pPTR0;
425	/** The GC virtual address of the two PAE page table. */
426	R0PTRTYPE(PX86PTPAE) paPaePTsR0;
427	} aPTs[1];
428	} PGMMAPPING;
429	/** Pointer to structure for tracking GC Mappings. */
430	typedef struct PGMMAPPING *PPGMMAPPING;
431
432
433	/**
434	* Physical page access handler structure.
435	*
436	* This is used to keep track of physical address ranges
437	* which are being monitored in some kind of way.
438	*/
439	typedef struct PGMPHYSHANDLER
440	{
441	AVLROGCPHYSNODECORE Core;
442	/** Access type. */
443	PGMPHYSHANDLERTYPE enmType;
444	/** Number of pages to update. */
445	uint32_t cPages;
446	/** Pointer to R3 callback function. */
447	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
448	/** User argument for R3 handlers. */
449	R3PTRTYPE(void *) pvUserR3;
450	/** Pointer to R0 callback function. */
451	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
452	/** User argument for R0 handlers. */
453	R0PTRTYPE(void *) pvUserR0;
454	/** Pointer to GC callback function. */
455	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
456	/** User argument for RC handlers. */
457	RCPTRTYPE(void *) pvUserRC;
458	/** Description / Name. For easing debugging. */
459	R3PTRTYPE(const char *) pszDesc;
460	#ifdef VBOX_WITH_STATISTICS
461	/** Profiling of this handler. */
462	STAMPROFILE Stat;
463	#endif
464	} PGMPHYSHANDLER;
465	/** Pointer to a physical page access handler structure. */
466	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
467
468
469	/**
470	* Cache node for the physical addresses covered by a virtual handler.
471	*/
472	typedef struct PGMPHYS2VIRTHANDLER
473	{
474	/** Core node for the tree based on physical ranges. */
475	AVLROGCPHYSNODECORE Core;
476	/** Offset from this struct to the PGMVIRTHANDLER structure. */
477	int32_t offVirtHandler;
478	/** Offset of the next alias relative to this one.
479	* Bit 0 is used for indicating whether we're in the tree.
480	* Bit 1 is used for indicating that we're the head node.
481	*/
482	int32_t offNextAlias;
483	} PGMPHYS2VIRTHANDLER;
484	/** Pointer to a phys to virtual handler structure. */
485	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
486
487	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
488	* node is in the tree. */
489	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
490	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
491	* node is in the head of an alias chain.
492	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
493	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
494	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
495	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
496
497
498	/**
499	* Virtual page access handler structure.
500	*
501	* This is used to keep track of virtual address ranges
502	* which are being monitored in some kind of way.
503	*/
504	typedef struct PGMVIRTHANDLER
505	{
506	/** Core node for the tree based on virtual ranges. */
507	AVLROGCPTRNODECORE Core;
508	/** Size of the range (in bytes). */
509	RTGCPTR cb;
510	/** Number of cache pages. */
511	uint32_t cPages;
512	/** Access type. */
513	PGMVIRTHANDLERTYPE enmType;
514	/** Pointer to the RC callback function. */
515	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
516	#if HC_ARCH_BITS == 64
517	RTRCPTR padding;
518	#endif
519	/** Pointer to the R3 callback function for invalidation. */
520	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
521	/** Pointer to the R3 callback function. */
522	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
523	/** Description / Name. For easing debugging. */
524	R3PTRTYPE(const char *) pszDesc;
525	#ifdef VBOX_WITH_STATISTICS
526	/** Profiling of this handler. */
527	STAMPROFILE Stat;
528	#endif
529	/** Array of cached physical addresses for the monitored ranged. */
530	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
531	} PGMVIRTHANDLER;
532	/** Pointer to a virtual page access handler structure. */
533	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
534
535
536	/**
537	* Page type.
538	*
539	* @remarks This enum has to fit in a 3-bit field (see PGMPAGE::u3Type).
540	* @remarks This is used in the saved state, so changes to it requires bumping
541	* the saved state version.
542	* @todo So, convert to \#defines!
543	*/
544	typedef enum PGMPAGETYPE
545	{
546	/** The usual invalid zero entry. */
547	PGMPAGETYPE_INVALID = 0,
548	/** RAM page. (RWX) */
549	PGMPAGETYPE_RAM,
550	/** MMIO2 page. (RWX) */
551	PGMPAGETYPE_MMIO2,
552	/** MMIO2 page aliased over an MMIO page. (RWX)
553	* See PGMHandlerPhysicalPageAlias(). */
554	PGMPAGETYPE_MMIO2_ALIAS_MMIO,
555	/** Shadowed ROM. (RWX) */
556	PGMPAGETYPE_ROM_SHADOW,
557	/** ROM page. (R-X) */
558	PGMPAGETYPE_ROM,
559	/** MMIO page. (---) */
560	PGMPAGETYPE_MMIO,
561	/** End of valid entries. */
562	PGMPAGETYPE_END
563	} PGMPAGETYPE;
564	AssertCompile(PGMPAGETYPE_END <= 7);
565
566	/** @name Page type predicates.
567	* @{ */
568	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
569	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
570	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
571	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
572	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
573	/** @} */
574
575
576	/**
577	* A Physical Guest Page tracking structure.
578	*
579	* The format of this structure is complicated because we have to fit a lot
580	* of information into as few bits as possible. The format is also subject
581	* to change (there is one comming up soon). Which means that for we'll be
582	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
583	* accessess to the structure.
584	*/
585	typedef struct PGMPAGE
586	{
587	/** The physical address and a whole lot of other stuff. All bits are used! */
588	RTHCPHYS HCPhysX;
589	/** The page state. */
590	uint32_t u2StateX : 2;
591	/** Flag indicating that a write monitored page was written to when set. */
592	uint32_t fWrittenToX : 1;
593	/** For later. */
594	uint32_t fSomethingElse : 1;
595	/** The Page ID.
596	* @todo Merge with HCPhysX once we've liberated HCPhysX of its stuff.
597	* The HCPhysX will then be 100% static. */
598	uint32_t idPageX : 28;
599	/** The page type (PGMPAGETYPE). */
600	uint32_t u3Type : 3;
601	/** The physical handler state (PGM_PAGE_HNDL_PHYS_STATE) /
602	uint32_t u2HandlerPhysStateX : 2;
603	/** The virtual handler state (PGM_PAGE_HNDL_VIRT_STATE) /
604	uint32_t u2HandlerVirtStateX : 2;
605	uint32_t u29B : 25;
606	} PGMPAGE;
607	AssertCompileSize(PGMPAGE, 16);
608	/** Pointer to a physical guest page. */
609	typedef PGMPAGE *PPGMPAGE;
610	/** Pointer to a const physical guest page. */
611	typedef const PGMPAGE *PCPGMPAGE;
612	/** Pointer to a physical guest page pointer. */
613	typedef PPGMPAGE *PPPGMPAGE;
614
615
616	/**
617	* Clears the page structure.
618	* @param pPage Pointer to the physical guest page tracking structure.
619	*/
620	#define PGM_PAGE_CLEAR(pPage) \
621	do { \
622	(pPage)->HCPhysX = 0; \
623	(pPage)->u2StateX = 0; \
624	(pPage)->fWrittenToX = 0; \
625	(pPage)->fSomethingElse = 0; \
626	(pPage)->idPageX = 0; \
627	(pPage)->u3Type = 0; \
628	(pPage)->u29B = 0; \
629	} while (0)
630
631	/**
632	* Initializes the page structure.
633	* @param pPage Pointer to the physical guest page tracking structure.
634	*/
635	#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
636	do { \
637	(pPage)->HCPhysX = (_HCPhys); \
638	(pPage)->u2StateX = (_uState); \
639	(pPage)->fWrittenToX = 0; \
640	(pPage)->fSomethingElse = 0; \
641	(pPage)->idPageX = (_idPage); \
642	/(pPage)->u3Type = (_uType); - later / \
643	PGM_PAGE_SET_TYPE(pPage, _uType); \
644	(pPage)->u29B = 0; \
645	} while (0)
646
647	/**
648	* Initializes the page structure of a ZERO page.
649	* @param pPage Pointer to the physical guest page tracking structure.
650	*/
651	#define PGM_PAGE_INIT_ZERO(pPage, pVM, _uType) \
652	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
653	/** Temporary hack. Replaced by PGM_PAGE_INIT_ZERO once the old code is kicked out. */
654	# define PGM_PAGE_INIT_ZERO_REAL(pPage, pVM, _uType) \
655	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
656
657
658	/** @name The Page state, PGMPAGE::u2StateX.
659	* @{ */
660	/** The zero page.
661	* This is a per-VM page that's never ever mapped writable. */
662	#define PGM_PAGE_STATE_ZERO 0
663	/** A allocated page.
664	* This is a per-VM page allocated from the page pool (or wherever
665	* we get MMIO2 pages from if the type is MMIO2).
666	*/
667	#define PGM_PAGE_STATE_ALLOCATED 1
668	/** A allocated page that's being monitored for writes.
669	* The shadow page table mappings are read-only. When a write occurs, the
670	* fWrittenTo member is set, the page remapped as read-write and the state
671	* moved back to allocated. */
672	#define PGM_PAGE_STATE_WRITE_MONITORED 2
673	/** The page is shared, aka. copy-on-write.
674	* This is a page that's shared with other VMs. */
675	#define PGM_PAGE_STATE_SHARED 3
676	/** @} */
677
678
679	/**
680	* Gets the page state.
681	* @returns page state (PGM_PAGE_STATE_*).
682	* @param pPage Pointer to the physical guest page tracking structure.
683	*/
684	#define PGM_PAGE_GET_STATE(pPage) ( (pPage)->u2StateX )
685
686	/**
687	* Sets the page state.
688	* @param pPage Pointer to the physical guest page tracking structure.
689	* @param _uState The new page state.
690	*/
691	#define PGM_PAGE_SET_STATE(pPage, _uState) \
692	do { (pPage)->u2StateX = (_uState); } while (0)
693
694
695	/**
696	* Gets the host physical address of the guest page.
697	* @returns host physical address (RTHCPHYS).
698	* @param pPage Pointer to the physical guest page tracking structure.
699	*/
700	#define PGM_PAGE_GET_HCPHYS(pPage) ( (pPage)->HCPhysX & UINT64_C(0x0000fffffffff000) )
701
702	/**
703	* Sets the host physical address of the guest page.
704	* @param pPage Pointer to the physical guest page tracking structure.
705	* @param _HCPhys The new host physical address.
706	*/
707	#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
708	do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0xffff000000000fff)) \
709	\| ((_HCPhys) & UINT64_C(0x0000fffffffff000)); } while (0)
710
711	/**
712	* Get the Page ID.
713	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
714	* @param pPage Pointer to the physical guest page tracking structure.
715	*/
716	#define PGM_PAGE_GET_PAGEID(pPage) ( (pPage)->idPageX )
717	/* later:
718	#define PGM_PAGE_GET_PAGEID(pPage) ( ((uint32_t)(pPage)->HCPhysX >> (48 - 12))
719	\| ((uint32_t)(pPage)->HCPhysX & 0xfff) )
720	*/
721	/**
722	* Sets the Page ID.
723	* @param pPage Pointer to the physical guest page tracking structure.
724	*/
725	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->idPageX = (_idPage); } while (0)
726	/* later:
727	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0x0000fffffffff000)) \
728	\| ((_idPage) & 0xfff) \
729	\| (((_idPage) & 0x0ffff000) << (48-12)); } while (0)
730	*/
731
732	/**
733	* Get the Chunk ID.
734	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
735	* @param pPage Pointer to the physical guest page tracking structure.
736	*/
737	#define PGM_PAGE_GET_CHUNKID(pPage) ( (pPage)->idPageX >> GMM_CHUNKID_SHIFT )
738	/* later:
739	#if GMM_CHUNKID_SHIFT == 12
740	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> 48) )
741	#elif GMM_CHUNKID_SHIFT > 12
742	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> (48 + (GMM_CHUNKID_SHIFT - 12)) )
743	#elif GMM_CHUNKID_SHIFT < 12
744	# define PGM_PAGE_GET_CHUNKID(pPage) ( ( (uint32_t)((pPage)->HCPhysX >> 48) << (12 - GMM_CHUNKID_SHIFT) ) \
745	\| ( (uint32_t)((pPage)->HCPhysX & 0xfff) >> GMM_CHUNKID_SHIFT ) )
746	#else
747	# error "GMM_CHUNKID_SHIFT isn't defined or something."
748	#endif
749	*/
750
751	/**
752	* Get the index of the page within the allocaiton chunk.
753	* @returns The page index.
754	* @param pPage Pointer to the physical guest page tracking structure.
755	*/
756	#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (pPage)->idPageX & GMM_PAGEID_IDX_MASK )
757	/* later:
758	#if GMM_CHUNKID_SHIFT <= 12
759	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & GMM_PAGEID_IDX_MASK) )
760	#else
761	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & 0xfff) \
762	\| ( (uint32_t)((pPage)->HCPhysX >> 48) & (RT_BIT_32(GMM_CHUNKID_SHIFT - 12) - 1) ) )
763	#endif
764	*/
765
766
767	/**
768	* Gets the page type.
769	* @returns The page type.
770	* @param pPage Pointer to the physical guest page tracking structure.
771	*/
772	#define PGM_PAGE_GET_TYPE(pPage) (pPage)->u3Type
773
774	/**
775	* Sets the page type.
776	* @param pPage Pointer to the physical guest page tracking structure.
777	* @param _enmType The new page type (PGMPAGETYPE).
778	*/
779	#define PGM_PAGE_SET_TYPE(pPage, _enmType) \
780	do { (pPage)->u3Type = (_enmType); } while (0)
781
782	/**
783	* Checks if the page is marked for MMIO.
784	* @returns true/false.
785	* @param pPage Pointer to the physical guest page tracking structure.
786	*/
787	#define PGM_PAGE_IS_MMIO(pPage) ( (pPage)->u3Type == PGMPAGETYPE_MMIO )
788
789	/**
790	* Checks if the page is backed by the ZERO page.
791	* @returns true/false.
792	* @param pPage Pointer to the physical guest page tracking structure.
793	*/
794	#define PGM_PAGE_IS_ZERO(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_ZERO )
795
796	/**
797	* Checks if the page is backed by a SHARED page.
798	* @returns true/false.
799	* @param pPage Pointer to the physical guest page tracking structure.
800	*/
801	#define PGM_PAGE_IS_SHARED(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_SHARED )
802
803
804	/**
805	* Marks the paget as written to (for GMM change monitoring).
806	* @param pPage Pointer to the physical guest page tracking structure.
807	*/
808	#define PGM_PAGE_SET_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 1; } while (0)
809
810	/**
811	* Clears the written-to indicator.
812	* @param pPage Pointer to the physical guest page tracking structure.
813	*/
814	#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 0; } while (0)
815
816	/**
817	* Checks if the page was marked as written-to.
818	* @returns true/false.
819	* @param pPage Pointer to the physical guest page tracking structure.
820	*/
821	#define PGM_PAGE_IS_WRITTEN_TO(pPage) ( (pPage)->fWrittenToX )
822
823
824	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateX).
825	*
826	* @remarks The values are assigned in order of priority, so we can calculate
827	* the correct state for a page with different handlers installed.
828	* @{ */
829	/** No handler installed. */
830	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
831	/** Monitoring is temporarily disabled. */
832	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
833	/** Write access is monitored. */
834	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
835	/** All access is monitored. */
836	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
837	/** @} */
838
839	/**
840	* Gets the physical access handler state of a page.
841	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
842	* @param pPage Pointer to the physical guest page tracking structure.
843	*/
844	#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) ( (pPage)->u2HandlerPhysStateX )
845
846	/**
847	* Sets the physical access handler state of a page.
848	* @param pPage Pointer to the physical guest page tracking structure.
849	* @param _uState The new state value.
850	*/
851	#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
852	do { (pPage)->u2HandlerPhysStateX = (_uState); } while (0)
853
854	/**
855	* Checks if the page has any physical access handlers, including temporariliy disabled ones.
856	* @returns true/false
857	* @param pPage Pointer to the physical guest page tracking structure.
858	*/
859	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE )
860
861	/**
862	* Checks if the page has any active physical access handlers.
863	* @returns true/false
864	* @param pPage Pointer to the physical guest page tracking structure.
865	*/
866	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
867
868
869	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateX).
870	*
871	* @remarks The values are assigned in order of priority, so we can calculate
872	* the correct state for a page with different handlers installed.
873	* @{ */
874	/** No handler installed. */
875	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
876	/* 1 is reserved so the lineup is identical with the physical ones. */
877	/** Write access is monitored. */
878	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
879	/** All access is monitored. */
880	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
881	/** @} */
882
883	/**
884	* Gets the virtual access handler state of a page.
885	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
886	* @param pPage Pointer to the physical guest page tracking structure.
887	*/
888	#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ( (pPage)->u2HandlerVirtStateX )
889
890	/**
891	* Sets the virtual access handler state of a page.
892	* @param pPage Pointer to the physical guest page tracking structure.
893	* @param _uState The new state value.
894	*/
895	#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
896	do { (pPage)->u2HandlerVirtStateX = (_uState); } while (0)
897
898	/**
899	* Checks if the page has any virtual access handlers.
900	* @returns true/false
901	* @param pPage Pointer to the physical guest page tracking structure.
902	*/
903	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ( (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
904
905	/**
906	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
907	* virtual handlers.
908	* @returns true/false
909	* @param pPage Pointer to the physical guest page tracking structure.
910	*/
911	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
912
913
914
915	/**
916	* Checks if the page has any access handlers, including temporarily disabled ones.
917	* @returns true/false
918	* @param pPage Pointer to the physical guest page tracking structure.
919	*/
920	#define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
921	( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE \
922	\|\| (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
923
924	/**
925	* Checks if the page has any active access handlers.
926	* @returns true/false
927	* @param pPage Pointer to the physical guest page tracking structure.
928	*/
929	#define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
930	( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
931	\|\| (pPage)->u2HandlerVirtStateX >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
932
933	/**
934	* Checks if the page has any active access handlers catching all accesses.
935	* @returns true/false
936	* @param pPage Pointer to the physical guest page tracking structure.
937	*/
938	#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
939	( (pPage)->u2HandlerPhysStateX == PGM_PAGE_HNDL_PHYS_STATE_ALL \
940	\|\| (pPage)->u2HandlerVirtStateX == PGM_PAGE_HNDL_VIRT_STATE_ALL )
941
942
943
944
945	/** @def PGM_PAGE_GET_TRACKING
946	* Gets the packed shadow page pool tracking data associated with a guest page.
947	* @returns uint16_t containing the data.
948	* @param pPage Pointer to the physical guest page tracking structure.
949	*/
950	#define PGM_PAGE_GET_TRACKING(pPage) \
951	( ((uint16_t )&(pPage)->HCPhysX + 3) )
952
953	/** @def PGM_PAGE_SET_TRACKING
954	* Sets the packed shadow page pool tracking data associated with a guest page.
955	* @param pPage Pointer to the physical guest page tracking structure.
956	* @param u16TrackingData The tracking data to store.
957	*/
958	#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
959	do { ((uint16_t )&(pPage)->HCPhysX + 3) = (u16TrackingData); } while (0)
960
961	/** @def PGM_PAGE_GET_TD_CREFS
962	* Gets the @a cRefs tracking data member.
963	* @returns cRefs.
964	* @param pPage Pointer to the physical guest page tracking structure.
965	*/
966	#define PGM_PAGE_GET_TD_CREFS(pPage) \
967	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
968
969	#define PGM_PAGE_GET_TD_IDX(pPage) \
970	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
971
972	/**
973	* Ram range for GC Phys to HC Phys conversion.
974	*
975	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
976	* conversions too, but we'll let MM handle that for now.
977	*
978	* This structure is used by linked lists in both GC and HC.
979	*/
980	typedef struct PGMRAMRANGE
981	{
982	/** Start of the range. Page aligned. */
983	RTGCPHYS GCPhys;
984	/** Size of the range. (Page aligned of course). */
985	RTGCPHYS cb;
986	/** Pointer to the next RAM range - for R3. */
987	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
988	/** Pointer to the next RAM range - for R0. */
989	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
990	/** Pointer to the next RAM range - for RC. */
991	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
992	/** PGM_RAM_RANGE_FLAGS_* flags. */
993	uint32_t fFlags;
994	/** Last address in the range (inclusive). Page aligned (-1). */
995	RTGCPHYS GCPhysLast;
996	/** Start of the HC mapping of the range. This is only used for MMIO2. */
997	R3PTRTYPE(void *) pvR3;
998	/** The range description. */
999	R3PTRTYPE(const char *) pszDesc;
1000	/** Pointer to self - R0 pointer. */
1001	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1002	/** Pointer to self - RC pointer. */
1003	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1004	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1005	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 1];
1006	/** Array of physical guest page tracking structures. */
1007	PGMPAGE aPages[1];
1008	} PGMRAMRANGE;
1009	/** Pointer to Ram range for GC Phys to HC Phys conversion. */
1010	typedef PGMRAMRANGE *PPGMRAMRANGE;
1011
1012	/** @name PGMRAMRANGE::fFlags
1013	* @{ */
1014	/** The RAM range is floating around as an independent guest mapping. */
1015	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1016	/** @} */
1017
1018
1019	/**
1020	* Per page tracking structure for ROM image.
1021	*
1022	* A ROM image may have a shadow page, in which case we may have
1023	* two pages backing it. This structure contains the PGMPAGE for
1024	* both while PGMRAMRANGE have a copy of the active one. It is
1025	* important that these aren't out of sync in any regard other
1026	* than page pool tracking data.
1027	*/
1028	typedef struct PGMROMPAGE
1029	{
1030	/** The page structure for the virgin ROM page. */
1031	PGMPAGE Virgin;
1032	/** The page structure for the shadow RAM page. */
1033	PGMPAGE Shadow;
1034	/** The current protection setting. */
1035	PGMROMPROT enmProt;
1036	/** Pad the structure size to a multiple of 8. */
1037	uint32_t u32Padding;
1038	} PGMROMPAGE;
1039	/** Pointer to a ROM page tracking structure. */
1040	typedef PGMROMPAGE *PPGMROMPAGE;
1041
1042
1043	/**
1044	* A registered ROM image.
1045	*
1046	* This is needed to keep track of ROM image since they generally
1047	* intrude into a PGMRAMRANGE. It also keeps track of additional
1048	* info like the two page sets (read-only virgin and read-write shadow),
1049	* the current state of each page.
1050	*
1051	* Because access handlers cannot easily be executed in a different
1052	* context, the ROM ranges needs to be accessible and in all contexts.
1053	*/
1054	typedef struct PGMROMRANGE
1055	{
1056	/** Pointer to the next range - R3. */
1057	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1058	/** Pointer to the next range - R0. */
1059	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1060	/** Pointer to the next range - RC. */
1061	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1062	/** Pointer alignment */
1063	RTRCPTR GCPtrAlignment;
1064	/** Address of the range. */
1065	RTGCPHYS GCPhys;
1066	/** Address of the last byte in the range. */
1067	RTGCPHYS GCPhysLast;
1068	/** Size of the range. */
1069	RTGCPHYS cb;
1070	/** The flags (PGMPHYS_ROM_FLAG_). /
1071	uint32_t fFlags;
1072	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1073	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 7 : 3];
1074	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1075	* This is used for strictness checks. */
1076	R3PTRTYPE(const void *) pvOriginal;
1077	/** The ROM description. */
1078	R3PTRTYPE(const char *) pszDesc;
1079	/** The per page tracking structures. */
1080	PGMROMPAGE aPages[1];
1081	} PGMROMRANGE;
1082	/** Pointer to a ROM range. */
1083	typedef PGMROMRANGE *PPGMROMRANGE;
1084
1085
1086	/**
1087	* A registered MMIO2 (= Device RAM) range.
1088	*
1089	* There are a few reason why we need to keep track of these
1090	* registrations. One of them is the deregistration & cleanup
1091	* stuff, while another is that the PGMRAMRANGE associated with
1092	* such a region may have to be removed from the ram range list.
1093	*
1094	* Overlapping with a RAM range has to be 100% or none at all. The
1095	* pages in the existing RAM range must not be ROM nor MMIO. A guru
1096	* meditation will be raised if a partial overlap or an overlap of
1097	* ROM pages is encountered. On an overlap we will free all the
1098	* existing RAM pages and put in the ram range pages instead.
1099	*/
1100	typedef struct PGMMMIO2RANGE
1101	{
1102	/** The owner of the range. (a device) */
1103	PPDMDEVINSR3 pDevInsR3;
1104	/** Pointer to the ring-3 mapping of the allocation. */
1105	RTR3PTR pvR3;
1106	/** Pointer to the next range - R3. */
1107	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1108	/** Whether it's mapped or not. */
1109	bool fMapped;
1110	/** Whether it's overlapping or not. */
1111	bool fOverlapping;
1112	/** The PCI region number.
1113	* @remarks This ASSUMES that nobody will ever really need to have multiple
1114	* PCI devices with matching MMIO region numbers on a single device. */
1115	uint8_t iRegion;
1116	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundrary. */
1117	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 1 : 5];
1118	/** The associated RAM range. */
1119	PGMRAMRANGE RamRange;
1120	} PGMMMIO2RANGE;
1121	/** Pointer to a MMIO2 range. */
1122	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1123
1124
1125
1126
1127	/**
1128	* PGMPhysRead/Write cache entry
1129	*/
1130	typedef struct PGMPHYSCACHEENTRY
1131	{
1132	/** R3 pointer to physical page. */
1133	R3PTRTYPE(uint8_t *) pbR3;
1134	/** GC Physical address for cache entry */
1135	RTGCPHYS GCPhys;
1136	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1137	RTGCPHYS u32Padding0; /*< alignment padding. /
1138	#endif
1139	} PGMPHYSCACHEENTRY;
1140
1141	/**
1142	* PGMPhysRead/Write cache to reduce REM memory access overhead
1143	*/
1144	typedef struct PGMPHYSCACHE
1145	{
1146	/** Bitmap of valid cache entries */
1147	uint64_t aEntries;
1148	/** Cache entries */
1149	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1150	} PGMPHYSCACHE;
1151
1152
1153	/** Pointer to an allocation chunk ring-3 mapping. */
1154	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1155	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1156	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1157
1158	/**
1159	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1160	*
1161	* The primary tree (Core) uses the chunk id as key.
1162	* The secondary tree (AgeCore) is used for ageing and uses ageing sequence number as key.
1163	*/
1164	typedef struct PGMCHUNKR3MAP
1165	{
1166	/** The key is the chunk id. */
1167	AVLU32NODECORE Core;
1168	/** The key is the ageing sequence number. */
1169	AVLLU32NODECORE AgeCore;
1170	/** The current age thingy. */
1171	uint32_t iAge;
1172	/** The current reference count. */
1173	uint32_t volatile cRefs;
1174	/** The current permanent reference count. */
1175	uint32_t volatile cPermRefs;
1176	/** The mapping address. */
1177	void *pv;
1178	} PGMCHUNKR3MAP;
1179
1180	/**
1181	* Allocation chunk ring-3 mapping TLB entry.
1182	*/
1183	typedef struct PGMCHUNKR3MAPTLBE
1184	{
1185	/** The chunk id. */
1186	uint32_t volatile idChunk;
1187	#if HC_ARCH_BITS == 64
1188	uint32_t u32Padding; /*< alignment padding. /
1189	#endif
1190	/** The chunk map. */
1191	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1192	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1193	#else
1194	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1195	#endif
1196	} PGMCHUNKR3MAPTLBE;
1197	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1198	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1199
1200	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1201	* @remark Must be a power of two value. */
1202	#define PGM_CHUNKR3MAPTLB_ENTRIES 32
1203
1204	/**
1205	* Allocation chunk ring-3 mapping TLB.
1206	*
1207	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1208	* At first glance this might look kinda odd since AVL trees are
1209	* supposed to give the most optimial lookup times of all trees
1210	* due to their balancing. However, take a tree with 1023 nodes
1211	* in it, that's 10 levels, meaning that most searches has to go
1212	* down 9 levels before they find what they want. This isn't fast
1213	* compared to a TLB hit. There is the factor of cache misses,
1214	* and of course the problem with trees and branch prediction.
1215	* This is why we use TLBs in front of most of the trees.
1216	*
1217	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1218	* difficult when we switch to the new inlined AVL trees (from kStuff).
1219	*/
1220	typedef struct PGMCHUNKR3MAPTLB
1221	{
1222	/** The TLB entries. */
1223	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1224	} PGMCHUNKR3MAPTLB;
1225
1226	/**
1227	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1228	* @returns Chunk TLB index.
1229	* @param idChunk The Chunk ID.
1230	*/
1231	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1232
1233
1234	/**
1235	* Ring-3 guest page mapping TLB entry.
1236	* @remarks used in ring-0 as well at the moment.
1237	*/
1238	typedef struct PGMPAGER3MAPTLBE
1239	{
1240	/** Address of the page. */
1241	RTGCPHYS volatile GCPhys;
1242	/** The guest page. */
1243	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1244	R3PTRTYPE(PPGMPAGE) volatile pPage;
1245	#else
1246	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1247	#endif
1248	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1249	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1250	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1251	#else
1252	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1253	#endif
1254	/** The address */
1255	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1256	R3PTRTYPE(void *) volatile pv;
1257	#else
1258	R3R0PTRTYPE(void *) volatile pv;
1259	#endif
1260	#if HC_ARCH_BITS == 32
1261	uint32_t u32Padding; /*< alignment padding. /
1262	#endif
1263	} PGMPAGER3MAPTLBE;
1264	/** Pointer to an entry in the HC physical TLB. */
1265	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1266
1267
1268	/** The number of entries in the ring-3 guest page mapping TLB.
1269	* @remarks The value must be a power of two. */
1270	#define PGM_PAGER3MAPTLB_ENTRIES 64
1271
1272	/**
1273	* Ring-3 guest page mapping TLB.
1274	* @remarks used in ring-0 as well at the moment.
1275	*/
1276	typedef struct PGMPAGER3MAPTLB
1277	{
1278	/** The TLB entries. */
1279	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1280	} PGMPAGER3MAPTLB;
1281	/** Pointer to the ring-3 guest page mapping TLB. */
1282	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1283
1284	/**
1285	* Calculates the index of the TLB entry for the specified guest page.
1286	* @returns Physical TLB index.
1287	* @param GCPhys The guest physical address.
1288	*/
1289	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1290
1291
1292	/**
1293	* Mapping cache usage set entry.
1294	*
1295	* @remarks 16-bit ints was choosen as the set is not expected to be used beyond
1296	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1297	* cache. If it's extended to include ring-3, well, then something will
1298	* have be changed here...
1299	*/
1300	typedef struct PGMMAPSETENTRY
1301	{
1302	/** The mapping cache index. */
1303	uint16_t iPage;
1304	/** The number of references.
1305	* The max is UINT16_MAX - 1. */
1306	uint16_t cRefs;
1307	#if HC_ARCH_BITS == 64
1308	uint32_t alignment;
1309	#endif
1310	/** Pointer to the page. */
1311	RTR0PTR pvPage;
1312	/** The physical address for this entry. */
1313	RTHCPHYS HCPhys;
1314	} PGMMAPSETENTRY;
1315	/** Pointer to a mapping cache usage set entry. */
1316	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1317
1318	/**
1319	* Mapping cache usage set.
1320	*
1321	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1322	* done during exits / traps. The set is
1323	*/
1324	typedef struct PGMMAPSET
1325	{
1326	/** The number of occupied entries.
1327	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1328	* dynamic mappings. */
1329	uint32_t cEntries;
1330	/** The start of the current subset.
1331	* This is UINT32_MAX if no subset is currently open. */
1332	uint32_t iSubset;
1333	/** The index of the current CPU, only valid if the set is open. */
1334	int32_t iCpu;
1335	#if HC_ARCH_BITS == 64
1336	uint32_t alignment;
1337	#endif
1338	/** The entries. */
1339	PGMMAPSETENTRY aEntries[64];
1340	/** HCPhys -> iEntry fast lookup table.
1341	* Use PGMMAPSET_HASH for hashing.
1342	* The entries may or may not be valid, check against cEntries. */
1343	uint8_t aiHashTable[128];
1344	} PGMMAPSET;
1345	/** Pointer to the mapping cache set. */
1346	typedef PGMMAPSET *PPGMMAPSET;
1347
1348	/** PGMMAPSET::cEntries value for a closed set. */
1349	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1350
1351	/** Hash function for aiHashTable. */
1352	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1353
1354	/** The max fill size (strict builds). */
1355	#define PGMMAPSET_MAX_FILL (64U * 80U / 100U)
1356
1357
1358	/** @name Context neutrual page mapper TLB.
1359	*
1360	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1361	* code is writting in a kind of context neutrual way. Time will show whether
1362	* this actually makes sense or not...
1363	*
1364	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1365	* context ends up using a global mapping cache on some platforms
1366	* (darwin).
1367	*
1368	* @{ */
1369	/** @typedef PPGMPAGEMAPTLB
1370	* The page mapper TLB pointer type for the current context. */
1371	/** @typedef PPGMPAGEMAPTLB
1372	* The page mapper TLB entry pointer type for the current context. */
1373	/** @typedef PPGMPAGEMAPTLB
1374	* The page mapper TLB entry pointer pointer type for the current context. */
1375	/** @def PGM_PAGEMAPTLB_ENTRIES
1376	* The number of TLB entries in the page mapper TLB for the current context. */
1377	/** @def PGM_PAGEMAPTLB_IDX
1378	* Calculate the TLB index for a guest physical address.
1379	* @returns The TLB index.
1380	* @param GCPhys The guest physical address. */
1381	/** @typedef PPGMPAGEMAP
1382	* Pointer to a page mapper unit for current context. */
1383	/** @typedef PPPGMPAGEMAP
1384	* Pointer to a page mapper unit pointer for current context. */
1385	#ifdef IN_RC
1386	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1387	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1388	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1389	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1390	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1391	typedef void * PPGMPAGEMAP;
1392	typedef void ** PPPGMPAGEMAP;
1393	//#elif IN_RING0
1394	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1395	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1396	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1397	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1398	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1399	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1400	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1401	#else
1402	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1403	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1404	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1405	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1406	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1407	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1408	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1409	#endif
1410	/** @} */
1411
1412
1413	/** @name PGM Pool Indexes.
1414	* Aka. the unique shadow page identifier.
1415	* @{ */
1416	/** NIL page pool IDX. */
1417	#define NIL_PGMPOOL_IDX 0
1418	/** The first normal index. */
1419	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1420	/** Page directory (32-bit root). */
1421	#define PGMPOOL_IDX_PD 1
1422	/** Page Directory Pointer Table (PAE root). */
1423	#define PGMPOOL_IDX_PDPT 2
1424	/** AMD64 CR3 level index.*/
1425	#define PGMPOOL_IDX_AMD64_CR3 3
1426	/** Nested paging root.*/
1427	#define PGMPOOL_IDX_NESTED_ROOT 4
1428	/** The first normal index. */
1429	#define PGMPOOL_IDX_FIRST 5
1430	/** The last valid index. (inclusive, 14 bits) */
1431	#define PGMPOOL_IDX_LAST 0x3fff
1432	/** @} */
1433
1434	/** The NIL index for the parent chain. */
1435	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1436
1437	/**
1438	* Node in the chain linking a shadowed page to it's parent (user).
1439	*/
1440	#pragma pack(1)
1441	typedef struct PGMPOOLUSER
1442	{
1443	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1444	uint16_t iNext;
1445	/** The user page index. */
1446	uint16_t iUser;
1447	/** Index into the user table. */
1448	uint32_t iUserTable;
1449	} PGMPOOLUSER, *PPGMPOOLUSER;
1450	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1451	#pragma pack()
1452
1453
1454	/** The NIL index for the phys ext chain. */
1455	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1456
1457	/**
1458	* Node in the chain of physical cross reference extents.
1459	* @todo Calling this an 'extent' is not quite right, find a better name.
1460	*/
1461	#pragma pack(1)
1462	typedef struct PGMPOOLPHYSEXT
1463	{
1464	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1465	uint16_t iNext;
1466	/** The user page index. */
1467	uint16_t aidx[3];
1468	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1469	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1470	#pragma pack()
1471
1472
1473	/**
1474	* The kind of page that's being shadowed.
1475	*/
1476	typedef enum PGMPOOLKIND
1477	{
1478	/** The virtual invalid 0 entry. */
1479	PGMPOOLKIND_INVALID = 0,
1480	/** The entry is free (=unused). */
1481	PGMPOOLKIND_FREE,
1482
1483	/** Shw: 32-bit page table; Gst: no paging */
1484	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1485	/** Shw: 32-bit page table; Gst: 32-bit page table. */
1486	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1487	/** Shw: 32-bit page table; Gst: 4MB page. */
1488	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1489	/** Shw: PAE page table; Gst: no paging */
1490	PGMPOOLKIND_PAE_PT_FOR_PHYS,
1491	/** Shw: PAE page table; Gst: 32-bit page table. */
1492	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1493	/** Shw: PAE page table; Gst: Half of a 4MB page. */
1494	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1495	/** Shw: PAE page table; Gst: PAE page table. */
1496	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1497	/** Shw: PAE page table; Gst: 2MB page. */
1498	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1499
1500	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1501	PGMPOOLKIND_32BIT_PD,
1502	/** Shw: 32-bit page directory. Gst: no paging. */
1503	PGMPOOLKIND_32BIT_PD_PHYS,
1504	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1505	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1506	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1507	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1508	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1509	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1510	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1511	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1512	/** Shw: PAE page directory; Gst: PAE page directory. */
1513	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1514	/** Shw: PAE page directory; Gst: no paging. */
1515	PGMPOOLKIND_PAE_PD_PHYS,
1516
1517	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1518	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1519	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1520	PGMPOOLKIND_PAE_PDPT,
1521	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1522	PGMPOOLKIND_PAE_PDPT_PHYS,
1523
1524	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1525	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1526	/** Shw: 64-bit page directory pointer table; Gst: no paging */
1527	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1528	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1529	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1530	/** Shw: 64-bit page directory table; Gst: no paging */
1531	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
1532
1533	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1534	PGMPOOLKIND_64BIT_PML4,
1535
1536	/** Shw: EPT page directory pointer table; Gst: no paging */
1537	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1538	/** Shw: EPT page directory table; Gst: no paging */
1539	PGMPOOLKIND_EPT_PD_FOR_PHYS,
1540	/** Shw: EPT page table; Gst: no paging */
1541	PGMPOOLKIND_EPT_PT_FOR_PHYS,
1542
1543	/** Shw: Root Nested paging table. */
1544	PGMPOOLKIND_ROOT_NESTED,
1545
1546	/** The last valid entry. */
1547	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1548	} PGMPOOLKIND;
1549
1550
1551	/**
1552	* The tracking data for a page in the pool.
1553	*/
1554	typedef struct PGMPOOLPAGE
1555	{
1556	/** AVL node code with the (R3) physical address of this page. */
1557	AVLOHCPHYSNODECORE Core;
1558	/** Pointer to the R3 mapping of the page. */
1559	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1560	R3PTRTYPE(void *) pvPageR3;
1561	#else
1562	R3R0PTRTYPE(void *) pvPageR3;
1563	#endif
1564	/** The guest physical address. */
1565	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
1566	uint32_t Alignment0;
1567	#endif
1568	RTGCPHYS GCPhys;
1569	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1570	uint8_t enmKind;
1571	uint8_t bPadding;
1572	/** The index of this page. */
1573	uint16_t idx;
1574	/** The next entry in the list this page currently resides in.
1575	* It's either in the free list or in the GCPhys hash. */
1576	uint16_t iNext;
1577	#ifdef PGMPOOL_WITH_USER_TRACKING
1578	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1579	uint16_t iUserHead;
1580	/** The number of present entries. */
1581	uint16_t cPresent;
1582	/** The first entry in the table which is present. */
1583	uint16_t iFirstPresent;
1584	#endif
1585	#ifdef PGMPOOL_WITH_MONITORING
1586	/** The number of modifications to the monitored page. */
1587	uint16_t cModifications;
1588	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
1589	uint16_t iModifiedNext;
1590	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
1591	uint16_t iModifiedPrev;
1592	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1593	uint16_t iMonitoredNext;
1594	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1595	uint16_t iMonitoredPrev;
1596	#endif
1597	#ifdef PGMPOOL_WITH_CACHE
1598	/** The next page in the age list. */
1599	uint16_t iAgeNext;
1600	/** The previous page in the age list. */
1601	uint16_t iAgePrev;
1602	#endif /* PGMPOOL_WITH_CACHE */
1603	/** Used to indicate that the page is zeroed. */
1604	bool fZeroed;
1605	/** Used to indicate that a PT has non-global entries. */
1606	bool fSeenNonGlobal;
1607	/** Used to indicate that we're monitoring writes to the guest page. */
1608	bool fMonitored;
1609	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1610	* (All pages are in the age list.) */
1611	bool fCached;
1612	/** This is used by the R3 access handlers when invoked by an async thread.
1613	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1614	bool volatile fReusedFlushPending;
1615	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
1616	bool fLocked;
1617	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
1618	/** Pointer to a const pool page. */
1619	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1620
1621
1622	#ifdef PGMPOOL_WITH_CACHE
1623	/** The hash table size. */
1624	# define PGMPOOL_HASH_SIZE 0x40
1625	/** The hash function. */
1626	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1627	#endif
1628
1629
1630	/**
1631	* The shadow page pool instance data.
1632	*
1633	* It's all one big allocation made at init time, except for the
1634	* pages that is. The user nodes follows immediatly after the
1635	* page structures.
1636	*/
1637	typedef struct PGMPOOL
1638	{
1639	/** The VM handle - R3 Ptr. */
1640	PVMR3 pVMR3;
1641	/** The VM handle - R0 Ptr. */
1642	PVMR0 pVMR0;
1643	/** The VM handle - RC Ptr. */
1644	PVMRC pVMRC;
1645	/** The max pool size. This includes the special IDs. */
1646	uint16_t cMaxPages;
1647	/** The current pool size. */
1648	uint16_t cCurPages;
1649	/** The head of the free page list. */
1650	uint16_t iFreeHead;
1651	/* Padding. */
1652	uint16_t u16Padding;
1653	#ifdef PGMPOOL_WITH_USER_TRACKING
1654	/** Head of the chain of free user nodes. */
1655	uint16_t iUserFreeHead;
1656	/** The number of user nodes we've allocated. */
1657	uint16_t cMaxUsers;
1658	/** The number of present page table entries in the entire pool. */
1659	uint32_t cPresent;
1660	/** Pointer to the array of user nodes - RC pointer. */
1661	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
1662	/** Pointer to the array of user nodes - R3 pointer. */
1663	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
1664	/** Pointer to the array of user nodes - R0 pointer. */
1665	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
1666	#endif /* PGMPOOL_WITH_USER_TRACKING */
1667	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1668	/** Head of the chain of free phys ext nodes. */
1669	uint16_t iPhysExtFreeHead;
1670	/** The number of user nodes we've allocated. */
1671	uint16_t cMaxPhysExts;
1672	/** Pointer to the array of physical xref extent - RC pointer. */
1673	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
1674	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
1675	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
1676	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
1677	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
1678	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1679	#ifdef PGMPOOL_WITH_CACHE
1680	/** Hash table for GCPhys addresses. */
1681	uint16_t aiHash[PGMPOOL_HASH_SIZE];
1682	/** The head of the age list. */
1683	uint16_t iAgeHead;
1684	/** The tail of the age list. */
1685	uint16_t iAgeTail;
1686	/** Set if the cache is enabled. */
1687	bool fCacheEnabled;
1688	#endif /* PGMPOOL_WITH_CACHE */
1689	#ifdef PGMPOOL_WITH_MONITORING
1690	/** Head of the list of modified pages. */
1691	uint16_t iModifiedHead;
1692	/** The current number of modified pages. */
1693	uint16_t cModifiedPages;
1694	/** Access handler, RC. */
1695	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
1696	/** Access handler, R0. */
1697	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
1698	/** Access handler, R3. */
1699	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
1700	/** The access handler description (HC ptr). */
1701	R3PTRTYPE(const char *) pszAccessHandler;
1702	#endif /* PGMPOOL_WITH_MONITORING */
1703	/** The number of pages currently in use. */
1704	uint16_t cUsedPages;
1705	#ifdef VBOX_WITH_STATISTICS
1706	/** The high wather mark for cUsedPages. */
1707	uint16_t cUsedPagesHigh;
1708	uint32_t Alignment1; /*< Align the next member on a 64-bit boundrary. /
1709	/** Profiling pgmPoolAlloc(). */
1710	STAMPROFILEADV StatAlloc;
1711	/** Profiling pgmPoolClearAll(). */
1712	STAMPROFILE StatClearAll;
1713	/** Profiling pgmPoolFlushAllInt(). */
1714	STAMPROFILE StatFlushAllInt;
1715	/** Profiling pgmPoolFlushPage(). */
1716	STAMPROFILE StatFlushPage;
1717	/** Profiling pgmPoolFree(). */
1718	STAMPROFILE StatFree;
1719	/** Profiling time spent zeroing pages. */
1720	STAMPROFILE StatZeroPage;
1721	# ifdef PGMPOOL_WITH_USER_TRACKING
1722	/** Profiling of pgmPoolTrackDeref. */
1723	STAMPROFILE StatTrackDeref;
1724	/** Profiling pgmTrackFlushGCPhysPT. */
1725	STAMPROFILE StatTrackFlushGCPhysPT;
1726	/** Profiling pgmTrackFlushGCPhysPTs. */
1727	STAMPROFILE StatTrackFlushGCPhysPTs;
1728	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
1729	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
1730	/** Number of times we've been out of user records. */
1731	STAMCOUNTER StatTrackFreeUpOneUser;
1732	# endif
1733	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1734	/** Profiling deref activity related tracking GC physical pages. */
1735	STAMPROFILE StatTrackDerefGCPhys;
1736	/** Number of linear searches for a HCPhys in the ram ranges. */
1737	STAMCOUNTER StatTrackLinearRamSearches;
1738	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
1739	STAMCOUNTER StamTrackPhysExtAllocFailures;
1740	# endif
1741	# ifdef PGMPOOL_WITH_MONITORING
1742	/** Profiling the RC/R0 access handler. */
1743	STAMPROFILE StatMonitorRZ;
1744	/** Times we've failed interpreting the instruction. */
1745	STAMCOUNTER StatMonitorRZEmulateInstr;
1746	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
1747	STAMPROFILE StatMonitorRZFlushPage;
1748	/** Times we've detected fork(). */
1749	STAMCOUNTER StatMonitorRZFork;
1750	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
1751	STAMPROFILE StatMonitorRZHandled;
1752	/** Times we've failed interpreting a patch code instruction. */
1753	STAMCOUNTER StatMonitorRZIntrFailPatch1;
1754	/** Times we've failed interpreting a patch code instruction during flushing. */
1755	STAMCOUNTER StatMonitorRZIntrFailPatch2;
1756	/** The number of times we've seen rep prefixes we can't handle. */
1757	STAMCOUNTER StatMonitorRZRepPrefix;
1758	/** Profiling the REP STOSD cases we've handled. */
1759	STAMPROFILE StatMonitorRZRepStosd;
1760
1761	/** Profiling the R3 access handler. */
1762	STAMPROFILE StatMonitorR3;
1763	/** Times we've failed interpreting the instruction. */
1764	STAMCOUNTER StatMonitorR3EmulateInstr;
1765	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
1766	STAMPROFILE StatMonitorR3FlushPage;
1767	/** Times we've detected fork(). */
1768	STAMCOUNTER StatMonitorR3Fork;
1769	/** Profiling the R3 access we've handled (except REP STOSD). */
1770	STAMPROFILE StatMonitorR3Handled;
1771	/** The number of times we've seen rep prefixes we can't handle. */
1772	STAMCOUNTER StatMonitorR3RepPrefix;
1773	/** Profiling the REP STOSD cases we've handled. */
1774	STAMPROFILE StatMonitorR3RepStosd;
1775	/** The number of times we're called in an async thread an need to flush. */
1776	STAMCOUNTER StatMonitorR3Async;
1777	/** The high wather mark for cModifiedPages. */
1778	uint16_t cModifiedPagesHigh;
1779	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundrary. /
1780	# endif
1781	# ifdef PGMPOOL_WITH_CACHE
1782	/** The number of cache hits. */
1783	STAMCOUNTER StatCacheHits;
1784	/** The number of cache misses. */
1785	STAMCOUNTER StatCacheMisses;
1786	/** The number of times we've got a conflict of 'kind' in the cache. */
1787	STAMCOUNTER StatCacheKindMismatches;
1788	/** Number of times we've been out of pages. */
1789	STAMCOUNTER StatCacheFreeUpOne;
1790	/** The number of cacheable allocations. */
1791	STAMCOUNTER StatCacheCacheable;
1792	/** The number of uncacheable allocations. */
1793	STAMCOUNTER StatCacheUncacheable;
1794	# endif
1795	#elif HC_ARCH_BITS == 64
1796	uint32_t Alignment3; /*< Align the next member on a 64-bit boundrary. /
1797	#endif
1798	/** The AVL tree for looking up a page by its HC physical address. */
1799	AVLOHCPHYSTREE HCPhysTree;
1800	uint32_t Alignment4; /*< Align the next member on a 64-bit boundrary. /
1801	/** Array of pages. (cMaxPages in length)
1802	* The Id is the index into thist array.
1803	*/
1804	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
1805	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
1806
1807
1808	/** @def PGMPOOL_PAGE_2_PTR
1809	* Maps a pool page pool into the current context.
1810	*
1811	* @returns VBox status code.
1812	* @param pVM The VM handle.
1813	* @param pPage The pool page.
1814	*
1815	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1816	* small page window employeed by that function. Be careful.
1817	* @remark There is no need to assert on the result.
1818	*/
1819	#if defined(IN_RC)
1820	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1821	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1822	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1823	#elif defined(VBOX_STRICT)
1824	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageStrict(pPage)
1825	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
1826	{
1827	Assert(pPage && pPage->pvPageR3);
1828	return pPage->pvPageR3;
1829	}
1830	#else
1831	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) ((pPage)->pvPageR3)
1832	#endif
1833
1834	/** @def PGMPOOL_PAGE_2_PTR_BY_PGM
1835	* Maps a pool page pool into the current context.
1836	*
1837	* @returns VBox status code.
1838	* @param pPGM Pointer to the PGM instance data.
1839	* @param pPage The pool page.
1840	*
1841	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1842	* small page window employeed by that function. Be careful.
1843	* @remark There is no need to assert on the result.
1844	*/
1845	#if defined(IN_RC)
1846	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1847	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1848	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1849	#else
1850	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGM2VM(pPGM), pPage)
1851	#endif
1852
1853	/** @def PGMPOOL_PAGE_2_PTR_BY_PGMCPU
1854	* Maps a pool page pool into the current context.
1855	*
1856	* @returns VBox status code.
1857	* @param pPGM Pointer to the PGMCPU instance data.
1858	* @param pPage The pool page.
1859	*
1860	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1861	* small page window employeed by that function. Be careful.
1862	* @remark There is no need to assert on the result.
1863	*/
1864	#if defined(IN_RC)
1865	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1866	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1867	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1868	#else
1869	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGMCPU2VM(pPGM), pPage)
1870	#endif
1871
1872
1873	/** @name Per guest page tracking data.
1874	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
1875	* is to use more bits for it and split it up later on. But for now we'll play
1876	* safe and change as little as possible.
1877	*
1878	* The 16-bit word has two parts:
1879	*
1880	* The first 14-bit forms the @a idx field. It is either the index of a page in
1881	* the shadow page pool, or and index into the extent list.
1882	*
1883	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
1884	* shadow page pool references to the page. If cRefs equals
1885	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
1886	* (misnomer) table and not the shadow page pool.
1887	*
1888	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
1889	* the 16-bit word.
1890	*
1891	* @{ */
1892	/** The shift count for getting to the cRefs part. */
1893	#define PGMPOOL_TD_CREFS_SHIFT 14
1894	/** The mask applied after shifting the tracking data down by
1895	* PGMPOOL_TD_CREFS_SHIFT. */
1896	#define PGMPOOL_TD_CREFS_MASK 0x3
1897	/** The cRef value used to indiciate that the idx is the head of a
1898	* physical cross reference list. */
1899	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
1900	/** The shift used to get idx. */
1901	#define PGMPOOL_TD_IDX_SHIFT 0
1902	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
1903	#define PGMPOOL_TD_IDX_MASK 0x3fff
1904	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
1905	* simply too many mappings of this page. */
1906	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
1907
1908	/** @def PGMPOOL_TD_MAKE
1909	* Makes a 16-bit tracking data word.
1910	*
1911	* @returns tracking data.
1912	* @param cRefs The @a cRefs field. Must be within bounds!
1913	* @param idx The @a idx field. Must also be within bounds! */
1914	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
1915
1916	/** @def PGMPOOL_TD_GET_CREFS
1917	* Get the @a cRefs field from a tracking data word.
1918	*
1919	* @returns The @a cRefs field
1920	* @param u16 The tracking data word. */
1921	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
1922
1923	/** @def PGMPOOL_TD_GET_IDX
1924	* Get the @a idx field from a tracking data word.
1925	*
1926	* @returns The @a idx field
1927	* @param u16 The tracking data word. */
1928	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
1929	/** @} */
1930
1931
1932	/**
1933	* Trees are using self relative offsets as pointers.
1934	* So, all its data, including the root pointer, must be in the heap for HC and GC
1935	* to have the same layout.
1936	*/
1937	typedef struct PGMTREES
1938	{
1939	/** Physical access handlers (AVL range+offsetptr tree). */
1940	AVLROGCPHYSTREE PhysHandlers;
1941	/** Virtual access handlers (AVL range + GC ptr tree). */
1942	AVLROGCPTRTREE VirtHandlers;
1943	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
1944	AVLROGCPHYSTREE PhysToVirtHandlers;
1945	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
1946	AVLROGCPTRTREE HyperVirtHandlers;
1947	} PGMTREES;
1948	/** Pointer to PGM trees. */
1949	typedef PGMTREES *PPGMTREES;
1950
1951
1952	/** @name Paging mode macros
1953	* @{ */
1954	#ifdef IN_RC
1955	# define PGM_CTX(a,b) a##RC##b
1956	# define PGM_CTX_STR(a,b) a "GC" b
1957	# define PGM_CTX_DECL(type) VMMRCDECL(type)
1958	#else
1959	# ifdef IN_RING3
1960	# define PGM_CTX(a,b) a##R3##b
1961	# define PGM_CTX_STR(a,b) a "R3" b
1962	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
1963	# else
1964	# define PGM_CTX(a,b) a##R0##b
1965	# define PGM_CTX_STR(a,b) a "R0" b
1966	# define PGM_CTX_DECL(type) VMMDECL(type)
1967	# endif
1968	#endif
1969
1970	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
1971	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
1972	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
1973	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
1974	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
1975	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
1976	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
1977	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
1978	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
1979	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
1980	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
1981	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
1982	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
1983	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
1984	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
1985	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
1986	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
1987
1988	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
1989	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
1990	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
1991	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
1992	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
1993	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
1994	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
1995	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
1996	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
1997	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
1998	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
1999	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2000	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2001	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2002	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2003	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2004	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2005
2006	/* Shw_Gst */
2007	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2008	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2009	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2010	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2011	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2012	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2013	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2014	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2015	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2016	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2017	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2018	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2019	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2020	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2021	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2022	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2023	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2024	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2025	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2026
2027	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2028	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2029	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2030	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2031	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2032	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2033	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2034	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2035	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2036	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2037	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2038	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2039	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2040	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2041	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2042	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2043	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2044	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2045	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2046	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2047	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2048	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2049	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2050	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2051	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2052	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2053	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2054	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2055	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2056	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2057	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2058	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2059	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2060	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2061	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2062	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2063	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2064
2065	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2066	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2067	/** @} */
2068
2069	/**
2070	* Data for each paging mode.
2071	*/
2072	typedef struct PGMMODEDATA
2073	{
2074	/** The guest mode type. */
2075	uint32_t uGstType;
2076	/** The shadow mode type. */
2077	uint32_t uShwType;
2078
2079	/** @name Function pointers for Shadow paging.
2080	* @{
2081	*/
2082	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2083	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2084	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2085	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2086
2087	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2088	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2089
2090	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2091	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2092	/** @} */
2093
2094	/** @name Function pointers for Guest paging.
2095	* @{
2096	*/
2097	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2098	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2099	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2100	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2101	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2102	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2103	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2104	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2105	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2106	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2107	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2108	/** @} */
2109
2110	/** @name Function pointers for Both Shadow and Guest paging.
2111	* @{
2112	*/
2113	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2114	/* no pfnR3BthTrap0eHandler */
2115	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2116	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2117	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2118	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2119	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2120	#ifdef VBOX_STRICT
2121	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2122	#endif
2123	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2124	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2125
2126	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2127	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2128	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2129	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2130	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2131	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2132	#ifdef VBOX_STRICT
2133	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2134	#endif
2135	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2136	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2137
2138	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2139	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2140	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2141	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2142	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2143	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2144	#ifdef VBOX_STRICT
2145	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2146	#endif
2147	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2148	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2149	/** @} */
2150	} PGMMODEDATA, *PPGMMODEDATA;
2151
2152
2153
2154	/**
2155	* Converts a PGM pointer into a VM pointer.
2156	* @returns Pointer to the VM structure the PGM is part of.
2157	* @param pPGM Pointer to PGM instance data.
2158	*/
2159	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2160
2161	/**
2162	* PGM Data (part of VM)
2163	*/
2164	typedef struct PGM
2165	{
2166	/** Offset to the VM structure. */
2167	RTINT offVM;
2168	/** Offset of the PGMCPU structure relative to VMCPU. */
2169	RTINT offVCpuPGM;
2170
2171	/** @cfgm{RamPreAlloc, boolean, false}
2172	* Indicates whether the base RAM should all be allocated before starting
2173	* the VM (default), or if it should be allocated when first written to.
2174	*/
2175	bool fRamPreAlloc;
2176	/** Alignment padding. */
2177	bool afAlignment0[7];
2178
2179	/** What needs syncing (PGM_SYNC_*).
2180	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2181	* PGMFlushTLB, and PGMR3Load. */
2182	RTUINT fGlobalSyncFlags;
2183
2184	/*
2185	* This will be redefined at least two more times before we're done, I'm sure.
2186	* The current code is only to get on with the coding.
2187	* - 2004-06-10: initial version, bird.
2188	* - 2004-07-02: 1st time, bird.
2189	* - 2004-10-18: 2nd time, bird.
2190	* - 2005-07-xx: 3rd time, bird.
2191	*/
2192
2193	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2194	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
2195	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2196	RCPTRTYPE(PX86PTEPAE) paDynPageMapPaePTEsGC;
2197
2198	/** The host paging mode. (This is what SUPLib reports.) */
2199	SUPPAGINGMODE enmHostMode;
2200
2201	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2202	RTGCPHYS GCPhys4MBPSEMask;
2203
2204	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2205	* This is sorted by physical address and contains no overlapping ranges. */
2206	R3PTRTYPE(PPGMRAMRANGE) pRamRangesR3;
2207	/** R0 pointer corresponding to PGM::pRamRangesR3. */
2208	R0PTRTYPE(PPGMRAMRANGE) pRamRangesR0;
2209	/** RC pointer corresponding to PGM::pRamRangesR3. */
2210	RCPTRTYPE(PPGMRAMRANGE) pRamRangesRC;
2211	RTRCPTR alignment4; /*< structure alignment. /
2212
2213	/** Pointer to the list of ROM ranges - for R3.
2214	* This is sorted by physical address and contains no overlapping ranges. */
2215	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2216	/** R0 pointer corresponding to PGM::pRomRangesR3. */
2217	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2218	/** RC pointer corresponding to PGM::pRomRangesR3. */
2219	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
2220	/** Alignment padding. */
2221	RTRCPTR GCPtrPadding2;
2222
2223	/** Pointer to the list of MMIO2 ranges - for R3.
2224	* Registration order. */
2225	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
2226
2227	/** PGM offset based trees - R3 Ptr. */
2228	R3PTRTYPE(PPGMTREES) pTreesR3;
2229	/** PGM offset based trees - R0 Ptr. */
2230	R0PTRTYPE(PPGMTREES) pTreesR0;
2231	/** PGM offset based trees - RC Ptr. */
2232	RCPTRTYPE(PPGMTREES) pTreesRC;
2233
2234	/** Linked list of GC mappings - for RC.
2235	* The list is sorted ascending on address.
2236	*/
2237	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
2238	/** Linked list of GC mappings - for HC.
2239	* The list is sorted ascending on address.
2240	*/
2241	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
2242	/** Linked list of GC mappings - for R0.
2243	* The list is sorted ascending on address.
2244	*/
2245	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
2246
2247	/** Pointer to the 5 page CR3 content mapping.
2248	* The first page is always the CR3 (in some form) while the 4 other pages
2249	* are used of the PDs in PAE mode. */
2250	RTGCPTR GCPtrCR3Mapping;
2251	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
2252	uint32_t u32Alignment;
2253	#endif
2254
2255	/** Indicates that PGMR3FinalizeMappings has been called and that further
2256	* PGMR3MapIntermediate calls will be rejected. */
2257	bool fFinalizedMappings;
2258	/** If set no conflict checks are required. (boolean) */
2259	bool fMappingsFixed;
2260	/** If set, then no mappings are put into the shadow page table. (boolean) */
2261	bool fDisableMappings;
2262	/** Size of fixed mapping */
2263	uint32_t cbMappingFixed;
2264	/** Base address (GC) of fixed mapping */
2265	RTGCPTR GCPtrMappingFixed;
2266	/** The address of the previous RAM range mapping. */
2267	RTGCPTR GCPtrPrevRamRangeMapping;
2268
2269	/** @name Intermediate Context
2270	* @{ */
2271	/** Pointer to the intermediate page directory - Normal. */
2272	R3PTRTYPE(PX86PD) pInterPD;
2273	/** Pointer to the intermedate page tables - Normal.
2274	* There are two page tables, one for the identity mapping and one for
2275	* the host context mapping (of the core code). */
2276	R3PTRTYPE(PX86PT) apInterPTs[2];
2277	/** Pointer to the intermedate page tables - PAE. */
2278	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
2279	/** Pointer to the intermedate page directory - PAE. */
2280	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
2281	/** Pointer to the intermedate page directory - PAE. */
2282	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
2283	/** Pointer to the intermedate page-map level 4 - AMD64. */
2284	R3PTRTYPE(PX86PML4) pInterPaePML4;
2285	/** Pointer to the intermedate page directory - AMD64. */
2286	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
2287	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
2288	RTHCPHYS HCPhysInterPD;
2289	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
2290	RTHCPHYS HCPhysInterPaePDPT;
2291	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
2292	RTHCPHYS HCPhysInterPaePML4;
2293	/** @} */
2294
2295	/** Base address of the dynamic page mapping area.
2296	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
2297	*/
2298	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
2299	/** The index of the last entry used in the dynamic page mapping area. */
2300	RTUINT iDynPageMapLast;
2301	/** Cache containing the last entries in the dynamic page mapping area.
2302	* The cache size is covering half of the mapping area. */
2303	RTHCPHYS aHCPhysDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT + 1)];
2304	/** Keep a lock counter for the full (!) mapping area. */
2305	uint32_t aLockedDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT)];
2306
2307	/** The address of the ring-0 mapping cache if we're making use of it. */
2308	RTR0PTR pvR0DynMapUsed;
2309	#if HC_ARCH_BITS == 32
2310	RTR0PTR R0PtrPadding0; /*< Alignment. /
2311	#endif
2312
2313	/** PGM critical section.
2314	* This protects the physical & virtual access handlers, ram ranges,
2315	* and the page flag updating (some of it anyway).
2316	*/
2317	PDMCRITSECT CritSect;
2318
2319	/** Pointer to SHW+GST mode data (function pointers).
2320	* The index into this table is made up from */
2321	R3PTRTYPE(PPGMMODEDATA) paModeData;
2322
2323	/** Shadow Page Pool - R3 Ptr. */
2324	R3PTRTYPE(PPGMPOOL) pPoolR3;
2325	/** Shadow Page Pool - R0 Ptr. */
2326	R0PTRTYPE(PPGMPOOL) pPoolR0;
2327	/** Shadow Page Pool - RC Ptr. */
2328	RCPTRTYPE(PPGMPOOL) pPoolRC;
2329
2330	/** We're not in a state which permits writes to guest memory.
2331	* (Only used in strict builds.) */
2332	bool fNoMorePhysWrites;
2333
2334	/** Flush the cache on the next access. */
2335	bool fPhysCacheFlushPending;
2336	/** @todo r=bird: Fix member names!*/
2337	/** PGMPhysRead cache */
2338	PGMPHYSCACHE pgmphysreadcache;
2339	/** PGMPhysWrite cache */
2340	PGMPHYSCACHE pgmphyswritecache;
2341
2342	/**
2343	* Data associated with managing the ring-3 mappings of the allocation chunks.
2344	*/
2345	struct
2346	{
2347	/** The chunk tree, ordered by chunk id. */
2348	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2349	R3PTRTYPE(PAVLU32NODECORE) pTree;
2350	#else
2351	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
2352	#endif
2353	/** The chunk mapping TLB. */
2354	PGMCHUNKR3MAPTLB Tlb;
2355	/** The number of mapped chunks. */
2356	uint32_t c;
2357	/** The maximum number of mapped chunks.
2358	* @cfgm PGM/MaxRing3Chunks */
2359	uint32_t cMax;
2360	/** The chunk age tree, ordered by ageing sequence number. */
2361	R3PTRTYPE(PAVLLU32NODECORE) pAgeTree;
2362	/** The current time. */
2363	uint32_t iNow;
2364	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
2365	uint32_t AgeingCountdown;
2366	} ChunkR3Map;
2367
2368	/**
2369	* The page mapping TLB for ring-3 and (for the time being) ring-0.
2370	*/
2371	PGMPAGER3MAPTLB PhysTlbHC;
2372
2373	/** @name The zero page.
2374	* @{ */
2375	/** The host physical address of the zero page. */
2376	RTHCPHYS HCPhysZeroPg;
2377	/** The ring-3 mapping of the zero page. */
2378	RTR3PTR pvZeroPgR3;
2379	/** The ring-0 mapping of the zero page. */
2380	RTR0PTR pvZeroPgR0;
2381	/** The GC mapping of the zero page. */
2382	RTGCPTR pvZeroPgRC;
2383	#if GC_ARCH_BITS != 32
2384	uint32_t u32ZeroAlignment; /*< Alignment padding. /
2385	#endif
2386	/** @}*/
2387
2388	/** The number of handy pages. */
2389	uint32_t cHandyPages;
2390	/**
2391	* Array of handy pages.
2392	*
2393	* This array is used in a two way communication between pgmPhysAllocPage
2394	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
2395	* an intermediary.
2396	*
2397	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
2398	* (The current size of 32 pages, means 128 KB of handy memory.)
2399	*/
2400	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
2401
2402	/** @name Error injection.
2403	* @{ */
2404	/** Inject handy page allocation errors pretending we're completely out of
2405	* memory. */
2406	bool volatile fErrInjHandyPages;
2407	/** Padding. */
2408	bool afReserved[7];
2409	/** @} */
2410
2411	/** @name Release Statistics
2412	* @{ */
2413	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero.) /
2414	uint32_t cPrivatePages; /*< The number of private pages. /
2415	uint32_t cSharedPages; /*< The number of shared pages. /
2416	uint32_t cZeroPages; /*< The number of zero backed pages. /
2417
2418	/** The number of times we were forced to change the hypervisor region location. */
2419	STAMCOUNTER cRelocations;
2420	/** @} */
2421
2422	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2423	/* R3 only: */
2424	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2425	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2426
2427	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2428	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2429	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2430	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2431	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2432	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2433	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2434	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2435	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2436	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2437	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2438	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2439	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2440	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2441	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2442	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2443	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2444	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2445	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2446	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2447	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2448	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2449
2450	/* RC only: */
2451	STAMCOUNTER StatRCDynMapCacheMisses; /*< RC: The number of dynamic page mapping cache hits /
2452	STAMCOUNTER StatRCDynMapCacheHits; /*< RC: The number of dynamic page mapping cache misses /
2453	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2454	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2455
2456	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2457	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2458	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2459	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2460	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2461	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2462	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2463	# endif
2464	#endif
2465	} PGM;
2466	/** Pointer to the PGM instance data. */
2467	typedef PGM *PPGM;
2468
2469
2470	/**
2471	* Converts a PGMCPU pointer into a VM pointer.
2472	* @returns Pointer to the VM structure the PGM is part of.
2473	* @param pPGM Pointer to PGMCPU instance data.
2474	*/
2475	#define PGMCPU2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2476
2477	/**
2478	* Converts a PGMCPU pointer into a PGM pointer.
2479	* @returns Pointer to the VM structure the PGM is part of.
2480	* @param pPGM Pointer to PGMCPU instance data.
2481	*/
2482	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char*)pPGMCpu - pPGMCpu->offPGM) )
2483
2484	/**
2485	* PGMCPU Data (part of VMCPU).
2486	*/
2487	typedef struct PGMCPU
2488	{
2489	/** Offset to the VM structure. */
2490	RTINT offVM;
2491	/** Offset to the VMCPU structure. */
2492	RTINT offVCpu;
2493	/** Offset of the PGM structure relative to VMCPU. */
2494	RTINT offPGM;
2495	RTINT uPadding0; /*< structure size alignment. /
2496
2497	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2498	/** Automatically tracked physical memory mapping set.
2499	* Ring-0 and strict raw-mode builds. */
2500	PGMMAPSET AutoSet;
2501	#endif
2502
2503	/** A20 gate mask.
2504	* Our current approach to A20 emulation is to let REM do it and don't bother
2505	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2506	* But whould need arrise, we'll subject physical addresses to this mask. */
2507	RTGCPHYS GCPhysA20Mask;
2508	/** A20 gate state - boolean! */
2509	bool fA20Enabled;
2510
2511	/** What needs syncing (PGM_SYNC_*).
2512	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2513	* PGMFlushTLB, and PGMR3Load. */
2514	RTUINT fSyncFlags;
2515
2516	/** The shadow paging mode. */
2517	PGMMODE enmShadowMode;
2518	/** The guest paging mode. */
2519	PGMMODE enmGuestMode;
2520
2521	/** The current physical address representing in the guest CR3 register. */
2522	RTGCPHYS GCPhysCR3;
2523
2524	/** @name 32-bit Guest Paging.
2525	* @{ */
2526	/** The guest's page directory, R3 pointer. */
2527	R3PTRTYPE(PX86PD) pGst32BitPdR3;
2528	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2529	/** The guest's page directory, R0 pointer. */
2530	R0PTRTYPE(PX86PD) pGst32BitPdR0;
2531	#endif
2532	/** The guest's page directory, static RC mapping. */
2533	RCPTRTYPE(PX86PD) pGst32BitPdRC;
2534	/** @} */
2535
2536	/** @name PAE Guest Paging.
2537	* @{ */
2538	/** The guest's page directory pointer table, static RC mapping. */
2539	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
2540	/** The guest's page directory pointer table, R3 pointer. */
2541	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
2542	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2543	/** The guest's page directory pointer table, R0 pointer. */
2544	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
2545	#endif
2546
2547	/** The guest's page directories, R3 pointers.
2548	* These are individual pointers and don't have to be adjecent.
2549	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2550	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
2551	/** The guest's page directories, R0 pointers.
2552	* Same restrictions as apGstPaePDsR3. */
2553	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2554	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
2555	#endif
2556	/** The guest's page directories, static GC mapping.
2557	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
2558	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2559	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
2560	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
2561	RTGCPHYS aGCPhysGstPaePDs[4];
2562	/** The physical addresses of the monitored guest page directories (PAE). */
2563	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
2564	/** @} */
2565
2566	/** @name AMD64 Guest Paging.
2567	* @{ */
2568	/** The guest's page directory pointer table, R3 pointer. */
2569	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
2570	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2571	/** The guest's page directory pointer table, R0 pointer. */
2572	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
2573	#endif
2574	/** @} */
2575
2576	/** Pointer to the page of the current active CR3 - R3 Ptr. */
2577	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
2578	/** Pointer to the page of the current active CR3 - R0 Ptr. */
2579	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
2580	/** Pointer to the page of the current active CR3 - RC Ptr. */
2581	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
2582	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
2583	uint32_t iShwUser;
2584	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
2585	uint32_t iShwUserTable;
2586	# if HC_ARCH_BITS == 64
2587	RTRCPTR alignment6; /*< structure size alignment. /
2588	# endif
2589	/** @} */
2590
2591	/** @name Function pointers for Shadow paging.
2592	* @{
2593	*/
2594	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2595	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2596	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2597	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2598
2599	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2600	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2601
2602	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2603	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2604
2605	/** @} */
2606
2607	/** @name Function pointers for Guest paging.
2608	* @{
2609	*/
2610	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2611	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2612	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2613	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2614	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2615	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2616	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2617	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2618	#if HC_ARCH_BITS == 64
2619	RTRCPTR alignment3; /*< structure size alignment. /
2620	#endif
2621
2622	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2623	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2624	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2625	/** @} */
2626
2627	/** @name Function pointers for Both Shadow and Guest paging.
2628	* @{
2629	*/
2630	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2631	/* no pfnR3BthTrap0eHandler */
2632	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2633	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2634	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2635	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2636	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2637	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2638	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2639	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2640
2641	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2642	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2643	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2644	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2645	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2646	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2647	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2648	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2649	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2650
2651	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2652	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2653	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2654	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2655	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2656	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2657	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2658	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2659	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2660	#if HC_ARCH_BITS == 64
2661	RTRCPTR alignment2; /*< structure size alignment. /
2662	#endif
2663	/** @} */
2664
2665	/** @name Release Statistics
2666	* @{ */
2667	/** The number of times the guest has switched mode since last reset or statistics reset. */
2668	STAMCOUNTER cGuestModeChanges;
2669	/** @} */
2670
2671	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2672	/** @name Statistics
2673	* @{ */
2674	/** RC: Which statistic this \#PF should be attributed to. */
2675	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
2676	RTRCPTR padding0;
2677	/** R0: Which statistic this \#PF should be attributed to. */
2678	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
2679	RTR0PTR padding1;
2680
2681	/* Common */
2682	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
2683	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
2684
2685	/* R0 only: */
2686	STAMCOUNTER StatR0DynMapMigrateInvlPg; /*< R0: invlpg in PGMDynMapMigrateAutoSet. /
2687	STAMPROFILE StatR0DynMapGCPageInl; /*< R0: Calls to pgmR0DynMapGCPageInlined. /
2688	STAMCOUNTER StatR0DynMapGCPageInlHits; /*< R0: Hash table lookup hits. /
2689	STAMCOUNTER StatR0DynMapGCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2690	STAMCOUNTER StatR0DynMapGCPageInlRamHits; /*< R0: 1st ram range hits. /
2691	STAMCOUNTER StatR0DynMapGCPageInlRamMisses; /*< R0: 1st ram range misses, takes slow path. /
2692	STAMPROFILE StatR0DynMapHCPageInl; /*< R0: Calls to pgmR0DynMapHCPageInlined. /
2693	STAMCOUNTER StatR0DynMapHCPageInlHits; /*< R0: Hash table lookup hits. /
2694	STAMCOUNTER StatR0DynMapHCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2695	STAMPROFILE StatR0DynMapHCPage; /*< R0: Calls to PGMDynMapHCPage. /
2696	STAMCOUNTER StatR0DynMapSetOptimize; /*< R0: Calls to pgmDynMapOptimizeAutoSet. /
2697	STAMCOUNTER StatR0DynMapSetSearchFlushes; /*< R0: Set search restorting to subset flushes. /
2698	STAMCOUNTER StatR0DynMapSetSearchHits; /*< R0: Set search hits. /
2699	STAMCOUNTER StatR0DynMapSetSearchMisses; /*< R0: Set search misses. /
2700	STAMCOUNTER StatR0DynMapPage; /*< R0: Calls to pgmR0DynMapPage. /
2701	STAMCOUNTER StatR0DynMapPageHits0; /*< R0: Hits at iPage+0. /
2702	STAMCOUNTER StatR0DynMapPageHits1; /*< R0: Hits at iPage+1. /
2703	STAMCOUNTER StatR0DynMapPageHits2; /*< R0: Hits at iPage+2. /
2704	STAMCOUNTER StatR0DynMapPageInvlPg; /*< R0: invlpg. /
2705	STAMCOUNTER StatR0DynMapPageSlow; /*< R0: Calls to pgmR0DynMapPageSlow. /
2706	STAMCOUNTER StatR0DynMapPageSlowLoopHits; /*< R0: Hits in the pgmR0DynMapPageSlow search loop. /
2707	STAMCOUNTER StatR0DynMapPageSlowLoopMisses; /*< R0: Misses in the pgmR0DynMapPageSlow search loop. /
2708	//STAMCOUNTER StatR0DynMapPageSlowLostHits; /*< R0: Lost hits. /
2709	STAMCOUNTER StatR0DynMapSubsets; /*< R0: Times PGMDynMapPushAutoSubset was called. /
2710	STAMCOUNTER StatR0DynMapPopFlushes; /*< R0: Times PGMDynMapPopAutoSubset flushes the subset. /
2711	STAMCOUNTER aStatR0DynMapSetSize[11]; /*< R0: Set size distribution. /
2712
2713	/* RZ only: */
2714	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
2715	STAMPROFILE StatRZTrap0eTimeCheckPageFault;
2716	STAMPROFILE StatRZTrap0eTimeSyncPT;
2717	STAMPROFILE StatRZTrap0eTimeMapping;
2718	STAMPROFILE StatRZTrap0eTimeOutOfSync;
2719	STAMPROFILE StatRZTrap0eTimeHandlers;
2720	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
2721	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
2722	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
2723	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
2724	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
2725	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
2726	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
2727	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
2728	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
2729	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
2730	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
2731	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
2732	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
2733	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
2734	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
2735	STAMCOUNTER StatRZTrap0eHandlersPhysical; /*< RC/R0: Number of traps due to physical access handlers. /
2736	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
2737	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
2738	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
2739	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
2740	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
2741	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: #PF err kind /
2742	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: #PF err kind /
2743	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: #PF err kind /
2744	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: #PF err kind /
2745	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: #PF err kind /
2746	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: #PF err kind /
2747	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: #PF err kind /
2748	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: #PF err kind /
2749	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: #PF err kind /
2750	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: #PF err kind /
2751	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: #PF err kind /
2752	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest #PFs. /
2753	STAMCOUNTER StatRZTrap0eGuestPFUnh; /*< RC/R0: Real guest #PF ending up at the end of the #PF code. /
2754	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest #PF to HMA or other mapping. /
2755	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
2756	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
2757	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the #PFs. /
2758	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
2759	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
2760	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
2761	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
2762	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
2763
2764	/* HC - R3 and (maybe) R0: */
2765
2766	/* RZ & R3: */
2767	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
2768	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
2769	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
2770	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
2771	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
2772	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
2773	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
2774	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
2775	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
2776	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
2777	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
2778	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
2779	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
2780	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
2781	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2782	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
2783	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
2784	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
2785	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2786	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2787	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
2788	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
2789	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
2790	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
2791	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
2792	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
2793	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
2794	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
2795	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
2796	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2797	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2798	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
2799	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2800	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2801	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2802	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2803	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
2804	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
2805	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2806	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2807	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2808	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2809	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
2810
2811	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
2812	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
2813	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
2814	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
2815	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
2816	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
2817	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
2818	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
2819	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
2820	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
2821	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
2822	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
2823	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
2824	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
2825	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2826	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
2827	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
2828	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
2829	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2830	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2831	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
2832	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
2833	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
2834	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
2835	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
2836	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
2837	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
2838	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
2839	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
2840	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2841	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
2842	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2843	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2844	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2845	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2846	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2847	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
2848	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
2849	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2850	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2851	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2852	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2853	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
2854	/** @} */
2855	#endif /* VBOX_WITH_STATISTICS */
2856	} PGMCPU;
2857	/** Pointer to the per-cpu PGM data. */
2858	typedef PGMCPU *PPGMCPU;
2859
2860
2861	/** @name PGM::fSyncFlags Flags
2862	* @{
2863	*/
2864	/** Updates the virtual access handler state bit in PGMPAGE. */
2865	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
2866	/** Always sync CR3. */
2867	#define PGM_SYNC_ALWAYS RT_BIT(1)
2868	/** Check monitoring on next CR3 (re)load and invalidate page.
2869	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
2870	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
2871	/** Check guest mapping in SyncCR3. */
2872	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
2873	/** Clear the page pool (a light weight flush). */
2874	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL RT_BIT(8)
2875	/** @} */
2876
2877
2878	__BEGIN_DECLS
2879
2880	int pgmLock(PVM pVM);
2881	void pgmUnlock(PVM pVM);
2882
2883	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
2884	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
2885	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
2886	void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping);
2887	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2888
2889	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
2890	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
2891	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage);
2892	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
2893	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
2894	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
2895	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
2896	#else
2897	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
2898	#endif
2899	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2900
2901
2902	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2903	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
2904	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2905	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2906	int pgmPhysPageMakeWritableUnlocked(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2907	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv);
2908	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
2909	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
2910	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
2911	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
2912	#ifdef IN_RING3
2913	void pgmR3PhysRelinkRamRanges(PVM pVM);
2914	int pgmR3PhysRamPreAllocate(PVM pVM);
2915	int pgmR3PhysRamReset(PVM pVM);
2916	int pgmR3PhysRomReset(PVM pVM);
2917	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
2918
2919	int pgmR3PoolInit(PVM pVM);
2920	void pgmR3PoolRelocate(PVM pVM);
2921	void pgmR3PoolReset(PVM pVM);
2922
2923	#endif /* IN_RING3 */
2924	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2925	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
2926	#endif
2927	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage);
2928	PPGMPOOLPAGE pgmPoolGetPageByHCPhys(PVM pVM, RTHCPHYS HCPhys);
2929	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
2930	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
2931	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2932	void pgmPoolFlushAll(PVM pVM);
2933	void pgmPoolClearAll(PVM pVM);
2934	int pgmPoolSyncCR3(PVM pVM);
2935	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs);
2936	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs);
2937	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt);
2938	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
2939	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
2940	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
2941	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
2942	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT);
2943	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage);
2944	#ifdef PGMPOOL_WITH_MONITORING
2945	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu);
2946	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2947	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2948	void pgmPoolMonitorModifiedClearAll(PVM pVM);
2949	int pgmPoolMonitorMonitorCR3(PPGMPOOL pPool, uint16_t idxRoot, RTGCPHYS GCPhysCR3);
2950	int pgmPoolMonitorUnmonitorCR3(PPGMPOOL pPool, uint16_t idxRoot);
2951	#endif
2952
2953	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
2954	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
2955
2956	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
2957	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
2958	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2959	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2960
2961	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2962	#ifndef IN_RC
2963	int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2964	#endif
2965	int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
2966
2967	PX86PD pgmGstLazyMap32BitPD(PPGMCPU pPGM);
2968	PX86PDPT pgmGstLazyMapPaePDPT(PPGMCPU pPGM);
2969	PX86PDPAE pgmGstLazyMapPaePD(PPGMCPU pPGM, uint32_t iPdpt);
2970	PX86PML4 pgmGstLazyMapPml4(PPGMCPU pPGM);
2971
2972	__END_DECLS
2973
2974
2975	/**
2976	* Gets the PGMRAMRANGE structure for a guest page.
2977	*
2978	* @returns Pointer to the RAM range on success.
2979	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
2980	*
2981	* @param pPGM PGM handle.
2982	* @param GCPhys The GC physical address.
2983	*/
2984	DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
2985	{
2986	/*
2987	* Optimize for the first range.
2988	*/
2989	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
2990	RTGCPHYS off = GCPhys - pRam->GCPhys;
2991	if (RT_UNLIKELY(off >= pRam->cb))
2992	{
2993	do
2994	{
2995	pRam = pRam->CTX_SUFF(pNext);
2996	if (RT_UNLIKELY(!pRam))
2997	break;
2998	off = GCPhys - pRam->GCPhys;
2999	} while (off >= pRam->cb);
3000	}
3001	return pRam;
3002	}
3003
3004
3005	/**
3006	* Gets the PGMPAGE structure for a guest page.
3007	*
3008	* @returns Pointer to the page on success.
3009	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3010	*
3011	* @param pPGM PGM handle.
3012	* @param GCPhys The GC physical address.
3013	*/
3014	DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
3015	{
3016	/*
3017	* Optimize for the first range.
3018	*/
3019	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3020	RTGCPHYS off = GCPhys - pRam->GCPhys;
3021	if (RT_UNLIKELY(off >= pRam->cb))
3022	{
3023	do
3024	{
3025	pRam = pRam->CTX_SUFF(pNext);
3026	if (RT_UNLIKELY(!pRam))
3027	return NULL;
3028	off = GCPhys - pRam->GCPhys;
3029	} while (off >= pRam->cb);
3030	}
3031	return &pRam->aPages[off >> PAGE_SHIFT];
3032	}
3033
3034
3035	/**
3036	* Gets the PGMPAGE structure for a guest page.
3037	*
3038	* Old Phys code: Will make sure the page is present.
3039	*
3040	* @returns VBox status code.
3041	* @retval VINF_SUCCESS and a valid *ppPage on success.
3042	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3043	*
3044	* @param pPGM PGM handle.
3045	* @param GCPhys The GC physical address.
3046	* @param ppPage Where to store the page poitner on success.
3047	*/
3048	DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
3049	{
3050	/*
3051	* Optimize for the first range.
3052	*/
3053	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3054	RTGCPHYS off = GCPhys - pRam->GCPhys;
3055	if (RT_UNLIKELY(off >= pRam->cb))
3056	{
3057	do
3058	{
3059	pRam = pRam->CTX_SUFF(pNext);
3060	if (RT_UNLIKELY(!pRam))
3061	{
3062	ppPage = NULL; / avoid incorrect and very annoying GCC warnings */
3063	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3064	}
3065	off = GCPhys - pRam->GCPhys;
3066	} while (off >= pRam->cb);
3067	}
3068	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3069	return VINF_SUCCESS;
3070	}
3071
3072
3073
3074
3075	/**
3076	* Gets the PGMPAGE structure for a guest page.
3077	*
3078	* Old Phys code: Will make sure the page is present.
3079	*
3080	* @returns VBox status code.
3081	* @retval VINF_SUCCESS and a valid *ppPage on success.
3082	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3083	*
3084	* @param pPGM PGM handle.
3085	* @param GCPhys The GC physical address.
3086	* @param ppPage Where to store the page poitner on success.
3087	* @param ppRamHint Where to read and store the ram list hint.
3088	* The caller initializes this to NULL before the call.
3089	*/
3090	DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
3091	{
3092	RTGCPHYS off;
3093	PPGMRAMRANGE pRam = *ppRamHint;
3094	if ( !pRam
3095	\|\| RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
3096	{
3097	pRam = pPGM->CTX_SUFF(pRamRanges);
3098	off = GCPhys - pRam->GCPhys;
3099	if (RT_UNLIKELY(off >= pRam->cb))
3100	{
3101	do
3102	{
3103	pRam = pRam->CTX_SUFF(pNext);
3104	if (RT_UNLIKELY(!pRam))
3105	{
3106	ppPage = NULL; / Kill the incorrect and extremely annoying GCC warnings. */
3107	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3108	}
3109	off = GCPhys - pRam->GCPhys;
3110	} while (off >= pRam->cb);
3111	}
3112	*ppRamHint = pRam;
3113	}
3114	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3115	return VINF_SUCCESS;
3116	}
3117
3118
3119	/**
3120	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3121	*
3122	* @returns Pointer to the page on success.
3123	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3124	*
3125	* @param pPGM PGM handle.
3126	* @param GCPhys The GC physical address.
3127	* @param ppRam Where to store the pointer to the PGMRAMRANGE.
3128	*/
3129	DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
3130	{
3131	/*
3132	* Optimize for the first range.
3133	*/
3134	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3135	RTGCPHYS off = GCPhys - pRam->GCPhys;
3136	if (RT_UNLIKELY(off >= pRam->cb))
3137	{
3138	do
3139	{
3140	pRam = pRam->CTX_SUFF(pNext);
3141	if (RT_UNLIKELY(!pRam))
3142	return NULL;
3143	off = GCPhys - pRam->GCPhys;
3144	} while (off >= pRam->cb);
3145	}
3146	*ppRam = pRam;
3147	return &pRam->aPages[off >> PAGE_SHIFT];
3148	}
3149
3150
3151	/**
3152	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3153	*
3154	* @returns Pointer to the page on success.
3155	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3156	*
3157	* @param pPGM PGM handle.
3158	* @param GCPhys The GC physical address.
3159	* @param ppPage Where to store the pointer to the PGMPAGE structure.
3160	* @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
3161	*/
3162	DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
3163	{
3164	/*
3165	* Optimize for the first range.
3166	*/
3167	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3168	RTGCPHYS off = GCPhys - pRam->GCPhys;
3169	if (RT_UNLIKELY(off >= pRam->cb))
3170	{
3171	do
3172	{
3173	pRam = pRam->CTX_SUFF(pNext);
3174	if (RT_UNLIKELY(!pRam))
3175	{
3176	ppRam = NULL; / Shut up silly GCC warnings. */
3177	ppPage = NULL; / ditto */
3178	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3179	}
3180	off = GCPhys - pRam->GCPhys;
3181	} while (off >= pRam->cb);
3182	}
3183	*ppRam = pRam;
3184	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3185	return VINF_SUCCESS;
3186	}
3187
3188
3189	/**
3190	* Convert GC Phys to HC Phys.
3191	*
3192	* @returns VBox status.
3193	* @param pPGM PGM handle.
3194	* @param GCPhys The GC physical address.
3195	* @param pHCPhys Where to store the corresponding HC physical address.
3196	*
3197	* @deprecated Doesn't deal with zero, shared or write monitored pages.
3198	* Avoid when writing new code!
3199	*/
3200	DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
3201	{
3202	PPGMPAGE pPage;
3203	int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
3204	if (RT_FAILURE(rc))
3205	return rc;
3206	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
3207	return VINF_SUCCESS;
3208	}
3209
3210	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3211
3212	/**
3213	* Inlined version of the ring-0 version of PGMDynMapHCPage that
3214	* optimizes access to pages already in the set.
3215	*
3216	* @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
3217	* @param pPGM Pointer to the PVM instance data.
3218	* @param HCPhys The physical address of the page.
3219	* @param ppv Where to store the mapping address.
3220	*/
3221	DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
3222	{
3223	PVM pVM = PGM2VM(pPGM);
3224	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3225	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3226
3227	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
3228	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3229	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3230
3231	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3232	unsigned iEntry = pSet->aiHashTable[iHash];
3233	if ( iEntry < pSet->cEntries
3234	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3235	{
3236	*ppv = pSet->aEntries[iEntry].pvPage;
3237	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
3238	}
3239	else
3240	{
3241	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
3242	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3243	}
3244
3245	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
3246	return VINF_SUCCESS;
3247	}
3248
3249
3250	/**
3251	* Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
3252	* access to pages already in the set.
3253	*
3254	* @returns See PGMDynMapGCPage.
3255	* @param pPGM Pointer to the PVM instance data.
3256	* @param HCPhys The physical address of the page.
3257	* @param ppv Where to store the mapping address.
3258	*/
3259	DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3260	{
3261	PVM pVM = PGM2VM(pPGM);
3262	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3263
3264	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3265	Assert(!(GCPhys & PAGE_OFFSET_MASK));
3266
3267	/*
3268	* Get the ram range.
3269	*/
3270	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3271	RTGCPHYS off = GCPhys - pRam->GCPhys;
3272	if (RT_UNLIKELY(off >= pRam->cb
3273	/** @todo \|\| page state stuff */))
3274	{
3275	/* This case is not counted into StatR0DynMapGCPageInl. */
3276	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3277	return PGMDynMapGCPage(pVM, GCPhys, ppv);
3278	}
3279
3280	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3281	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3282
3283	/*
3284	* pgmR0DynMapHCPageInlined with out stats.
3285	*/
3286	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3287	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3288	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3289
3290	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3291	unsigned iEntry = pSet->aiHashTable[iHash];
3292	if ( iEntry < pSet->cEntries
3293	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3294	{
3295	*ppv = pSet->aEntries[iEntry].pvPage;
3296	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3297	}
3298	else
3299	{
3300	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3301	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3302	}
3303
3304	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3305	return VINF_SUCCESS;
3306	}
3307
3308	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3309	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3310
3311	/**
3312	* Maps the page into current context (RC and maybe R0).
3313	*
3314	* @returns pointer to the mapping.
3315	* @param pVM Pointer to the PGM instance data.
3316	* @param pPage The page.
3317	*/
3318	DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
3319	{
3320	if (pPage->idx >= PGMPOOL_IDX_FIRST)
3321	{
3322	Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
3323	void *pv;
3324	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3325	pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
3326	# else
3327	PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
3328	# endif
3329	return pv;
3330	}
3331	AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
3332	}
3333
3334	/**
3335	* Temporarily maps one host page specified by HC physical address, returning
3336	* pointer within the page.
3337	*
3338	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
3339	* reused after 8 mappings (or perhaps a few more if you score with the cache).
3340	*
3341	* @returns The address corresponding to HCPhys.
3342	* @param pPGM Pointer to the PVM instance data.
3343	* @param HCPhys HC Physical address of the page.
3344	*/
3345	DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
3346	{
3347	void *pv;
3348	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3349	pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3350	# else
3351	PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3352	# endif
3353	pv = (void *)((uintptr_t)pv \| (HCPhys & PAGE_OFFSET_MASK));
3354	return pv;
3355	}
3356
3357	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 \|\| IN_RC */
3358	#ifndef IN_RC
3359
3360	/**
3361	* Queries the Physical TLB entry for a physical guest page,
3362	* attempting to load the TLB entry if necessary.
3363	*
3364	* @returns VBox status code.
3365	* @retval VINF_SUCCESS on success
3366	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3367	*
3368	* @param pPGM The PGM instance handle.
3369	* @param GCPhys The address of the guest page.
3370	* @param ppTlbe Where to store the pointer to the TLB entry.
3371	*/
3372	DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3373	{
3374	int rc;
3375	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3376	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3377	{
3378	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3379	rc = VINF_SUCCESS;
3380	}
3381	else
3382	rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
3383	*ppTlbe = pTlbe;
3384	return rc;
3385	}
3386
3387
3388	/**
3389	* Queries the Physical TLB entry for a physical guest page,
3390	* attempting to load the TLB entry if necessary.
3391	*
3392	* @returns VBox status code.
3393	* @retval VINF_SUCCESS on success
3394	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3395	*
3396	* @param pPGM The PGM instance handle.
3397	* @param pPage Pointer to the PGMPAGE structure corresponding to
3398	* GCPhys.
3399	* @param GCPhys The address of the guest page.
3400	* @param ppTlbe Where to store the pointer to the TLB entry.
3401	*/
3402	DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3403	{
3404	int rc;
3405	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3406	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3407	{
3408	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3409	rc = VINF_SUCCESS;
3410	}
3411	else
3412	rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
3413	*ppTlbe = pTlbe;
3414	return rc;
3415	}
3416
3417	#endif /* !IN_RC */
3418
3419	/**
3420	* Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
3421	* Takes PSE-36 into account.
3422	*
3423	* @returns guest physical address
3424	* @param pPGM Pointer to the PGM instance data.
3425	* @param Pde Guest Pde
3426	*/
3427	DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
3428	{
3429	RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
3430	GCPhys \|= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
3431
3432	return GCPhys & pPGM->GCPhys4MBPSEMask;
3433	}
3434
3435
3436	/**
3437	* Gets the page directory entry for the specified address (32-bit paging).
3438	*
3439	* @returns The page directory entry in question.
3440	* @param pPGM Pointer to the PGM instance data.
3441	* @param GCPtr The address.
3442	*/
3443	DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3444	{
3445	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3446	PCX86PD pGuestPD = NULL;
3447	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3448	if (RT_FAILURE(rc))
3449	{
3450	X86PDE ZeroPde = {0};
3451	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
3452	}
3453	#else
3454	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3455	# ifdef IN_RING3
3456	if (!pGuestPD)
3457	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3458	# endif
3459	#endif
3460	return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3461	}
3462
3463
3464	/**
3465	* Gets the address of a specific page directory entry (32-bit paging).
3466	*
3467	* @returns Pointer the page directory entry in question.
3468	* @param pPGM Pointer to the PGM instance data.
3469	* @param GCPtr The address.
3470	*/
3471	DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3472	{
3473	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3474	PX86PD pGuestPD = NULL;
3475	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3476	AssertRCReturn(rc, NULL);
3477	#else
3478	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3479	# ifdef IN_RING3
3480	if (!pGuestPD)
3481	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3482	# endif
3483	#endif
3484	return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3485	}
3486
3487
3488	/**
3489	* Gets the address the guest page directory (32-bit paging).
3490	*
3491	* @returns Pointer the page directory entry in question.
3492	* @param pPGM Pointer to the PGM instance data.
3493	*/
3494	DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
3495	{
3496	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3497	PX86PD pGuestPD = NULL;
3498	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3499	AssertRCReturn(rc, NULL);
3500	#else
3501	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3502	# ifdef IN_RING3
3503	if (!pGuestPD)
3504	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3505	# endif
3506	#endif
3507	return pGuestPD;
3508	}
3509
3510
3511	/**
3512	* Gets the guest page directory pointer table.
3513	*
3514	* @returns Pointer to the page directory in question.
3515	* @returns NULL if the page directory is not present or on an invalid page.
3516	* @param pPGM Pointer to the PGM instance data.
3517	*/
3518	DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
3519	{
3520	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3521	PX86PDPT pGuestPDPT = NULL;
3522	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3523	AssertRCReturn(rc, NULL);
3524	#else
3525	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3526	# ifdef IN_RING3
3527	if (!pGuestPDPT)
3528	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3529	# endif
3530	#endif
3531	return pGuestPDPT;
3532	}
3533
3534
3535	/**
3536	* Gets the guest page directory pointer table entry for the specified address.
3537	*
3538	* @returns Pointer to the page directory in question.
3539	* @returns NULL if the page directory is not present or on an invalid page.
3540	* @param pPGM Pointer to the PGM instance data.
3541	* @param GCPtr The address.
3542	*/
3543	DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3544	{
3545	AssertGCPtr32(GCPtr);
3546
3547	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3548	PX86PDPT pGuestPDPT = 0;
3549	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3550	AssertRCReturn(rc, 0);
3551	#else
3552	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3553	# ifdef IN_RING3
3554	if (!pGuestPDPT)
3555	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3556	# endif
3557	#endif
3558	return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
3559	}
3560
3561
3562	/**
3563	* Gets the page directory for the specified address.
3564	*
3565	* @returns Pointer to the page directory in question.
3566	* @returns NULL if the page directory is not present or on an invalid page.
3567	* @param pPGM Pointer to the PGM instance data.
3568	* @param GCPtr The address.
3569	*/
3570	DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
3571	{
3572	AssertGCPtr32(GCPtr);
3573
3574	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3575	AssertReturn(pGuestPDPT, NULL);
3576	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3577	if (pGuestPDPT->a[iPdpt].n.u1Present)
3578	{
3579	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3580	PX86PDPAE pGuestPD = NULL;
3581	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3582	AssertRCReturn(rc, NULL);
3583	#else
3584	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3585	if ( !pGuestPD
3586	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3587	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3588	#endif
3589	return pGuestPD;
3590	/* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
3591	}
3592	return NULL;
3593	}
3594
3595
3596	/**
3597	* Gets the page directory entry for the specified address.
3598	*
3599	* @returns Pointer to the page directory entry in question.
3600	* @returns NULL if the page directory is not present or on an invalid page.
3601	* @param pPGM Pointer to the PGM instance data.
3602	* @param GCPtr The address.
3603	*/
3604	DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3605	{
3606	AssertGCPtr32(GCPtr);
3607
3608	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3609	AssertReturn(pGuestPDPT, NULL);
3610	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3611	if (pGuestPDPT->a[iPdpt].n.u1Present)
3612	{
3613	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3614	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3615	PX86PDPAE pGuestPD = NULL;
3616	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3617	AssertRCReturn(rc, NULL);
3618	#else
3619	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3620	if ( !pGuestPD
3621	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3622	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3623	#endif
3624	return &pGuestPD->a[iPD];
3625	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
3626	}
3627	return NULL;
3628	}
3629
3630
3631	/**
3632	* Gets the page directory entry for the specified address.
3633	*
3634	* @returns The page directory entry in question.
3635	* @returns A non-present entry if the page directory is not present or on an invalid page.
3636	* @param pPGM Pointer to the PGM instance data.
3637	* @param GCPtr The address.
3638	*/
3639	DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3640	{
3641	AssertGCPtr32(GCPtr);
3642	X86PDEPAE ZeroPde = {0};
3643	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3644	if (RT_LIKELY(pGuestPDPT))
3645	{
3646	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3647	if (pGuestPDPT->a[iPdpt].n.u1Present)
3648	{
3649	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3650	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3651	PX86PDPAE pGuestPD = NULL;
3652	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3653	AssertRCReturn(rc, ZeroPde);
3654	#else
3655	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3656	if ( !pGuestPD
3657	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3658	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3659	#endif
3660	return pGuestPD->a[iPD];
3661	}
3662	}
3663	return ZeroPde;
3664	}
3665
3666
3667	/**
3668	* Gets the page directory pointer table entry for the specified address
3669	* and returns the index into the page directory
3670	*
3671	* @returns Pointer to the page directory in question.
3672	* @returns NULL if the page directory is not present or on an invalid page.
3673	* @param pPGM Pointer to the PGM instance data.
3674	* @param GCPtr The address.
3675	* @param piPD Receives the index into the returned page directory
3676	* @param pPdpe Receives the page directory pointer entry. Optional.
3677	*/
3678	DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
3679	{
3680	AssertGCPtr32(GCPtr);
3681
3682	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3683	AssertReturn(pGuestPDPT, NULL);
3684	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3685	if (pPdpe)
3686	*pPdpe = pGuestPDPT->a[iPdpt];
3687	if (pGuestPDPT->a[iPdpt].n.u1Present)
3688	{
3689	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3690	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3691	PX86PDPAE pGuestPD = NULL;
3692	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3693	AssertRCReturn(rc, NULL);
3694	#else
3695	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3696	if ( !pGuestPD
3697	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3698	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3699	#endif
3700	*piPD = iPD;
3701	return pGuestPD;
3702	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
3703	}
3704	return NULL;
3705	}
3706
3707	#ifndef IN_RC
3708
3709	/**
3710	* Gets the page map level-4 pointer for the guest.
3711	*
3712	* @returns Pointer to the PML4 page.
3713	* @param pPGM Pointer to the PGM instance data.
3714	*/
3715	DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
3716	{
3717	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3718	PX86PML4 pGuestPml4;
3719	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3720	AssertRCReturn(rc, NULL);
3721	#else
3722	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3723	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3724	if (!pGuestPml4)
3725	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3726	# endif
3727	Assert(pGuestPml4);
3728	#endif
3729	return pGuestPml4;
3730	}
3731
3732
3733	/**
3734	* Gets the pointer to a page map level-4 entry.
3735	*
3736	* @returns Pointer to the PML4 entry.
3737	* @param pPGM Pointer to the PGM instance data.
3738	* @param iPml4 The index.
3739	*/
3740	DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
3741	{
3742	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3743	PX86PML4 pGuestPml4;
3744	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3745	AssertRCReturn(rc, NULL);
3746	#else
3747	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3748	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3749	if (!pGuestPml4)
3750	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3751	# endif
3752	Assert(pGuestPml4);
3753	#endif
3754	return &pGuestPml4->a[iPml4];
3755	}
3756
3757
3758	/**
3759	* Gets a page map level-4 entry.
3760	*
3761	* @returns The PML4 entry.
3762	* @param pPGM Pointer to the PGM instance data.
3763	* @param iPml4 The index.
3764	*/
3765	DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
3766	{
3767	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3768	PX86PML4 pGuestPml4;
3769	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3770	if (RT_FAILURE(rc))
3771	{
3772	X86PML4E ZeroPml4e = {0};
3773	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
3774	}
3775	#else
3776	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3777	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3778	if (!pGuestPml4)
3779	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3780	# endif
3781	Assert(pGuestPml4);
3782	#endif
3783	return pGuestPml4->a[iPml4];
3784	}
3785
3786
3787	/**
3788	* Gets the page directory pointer entry for the specified address.
3789	*
3790	* @returns Pointer to the page directory pointer entry in question.
3791	* @returns NULL if the page directory is not present or on an invalid page.
3792	* @param pPGM Pointer to the PGM instance data.
3793	* @param GCPtr The address.
3794	* @param ppPml4e Page Map Level-4 Entry (out)
3795	*/
3796	DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
3797	{
3798	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3799	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3800	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3801	if (pPml4e->n.u1Present)
3802	{
3803	PX86PDPT pPdpt;
3804	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
3805	AssertRCReturn(rc, NULL);
3806
3807	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3808	return &pPdpt->a[iPdpt];
3809	}
3810	return NULL;
3811	}
3812
3813
3814	/**
3815	* Gets the page directory entry for the specified address.
3816	*
3817	* @returns The page directory entry in question.
3818	* @returns A non-present entry if the page directory is not present or on an invalid page.
3819	* @param pPGM Pointer to the PGM instance data.
3820	* @param GCPtr The address.
3821	* @param ppPml4e Page Map Level-4 Entry (out)
3822	* @param pPdpe Page directory pointer table entry (out)
3823	*/
3824	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
3825	{
3826	X86PDEPAE ZeroPde = {0};
3827	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3828	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3829	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3830	if (pPml4e->n.u1Present)
3831	{
3832	PCX86PDPT pPdptTemp;
3833	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3834	AssertRCReturn(rc, ZeroPde);
3835
3836	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3837	*pPdpe = pPdptTemp->a[iPdpt];
3838	if (pPdptTemp->a[iPdpt].n.u1Present)
3839	{
3840	PCX86PDPAE pPD;
3841	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3842	AssertRCReturn(rc, ZeroPde);
3843
3844	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3845	return pPD->a[iPD];
3846	}
3847	}
3848
3849	return ZeroPde;
3850	}
3851
3852
3853	/**
3854	* Gets the page directory entry for the specified address.
3855	*
3856	* @returns The page directory entry in question.
3857	* @returns A non-present entry if the page directory is not present or on an invalid page.
3858	* @param pPGM Pointer to the PGM instance data.
3859	* @param GCPtr The address.
3860	*/
3861	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3862	{
3863	X86PDEPAE ZeroPde = {0};
3864	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3865	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3866	if (pGuestPml4->a[iPml4].n.u1Present)
3867	{
3868	PCX86PDPT pPdptTemp;
3869	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3870	AssertRCReturn(rc, ZeroPde);
3871
3872	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3873	if (pPdptTemp->a[iPdpt].n.u1Present)
3874	{
3875	PCX86PDPAE pPD;
3876	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3877	AssertRCReturn(rc, ZeroPde);
3878
3879	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3880	return pPD->a[iPD];
3881	}
3882	}
3883	return ZeroPde;
3884	}
3885
3886
3887	/**
3888	* Gets the page directory entry for the specified address.
3889	*
3890	* @returns Pointer to the page directory entry in question.
3891	* @returns NULL if the page directory is not present or on an invalid page.
3892	* @param pPGM Pointer to the PGM instance data.
3893	* @param GCPtr The address.
3894	*/
3895	DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3896	{
3897	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3898	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3899	if (pGuestPml4->a[iPml4].n.u1Present)
3900	{
3901	PCX86PDPT pPdptTemp;
3902	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3903	AssertRCReturn(rc, NULL);
3904
3905	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3906	if (pPdptTemp->a[iPdpt].n.u1Present)
3907	{
3908	PX86PDPAE pPD;
3909	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3910	AssertRCReturn(rc, NULL);
3911
3912	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3913	return &pPD->a[iPD];
3914	}
3915	}
3916	return NULL;
3917	}
3918
3919
3920	/**
3921	* Gets the GUEST page directory pointer for the specified address.
3922	*
3923	* @returns The page directory in question.
3924	* @returns NULL if the page directory is not present or on an invalid page.
3925	* @param pPGM Pointer to the PGM instance data.
3926	* @param GCPtr The address.
3927	* @param ppPml4e Page Map Level-4 Entry (out)
3928	* @param pPdpe Page directory pointer table entry (out)
3929	* @param piPD Receives the index into the returned page directory
3930	*/
3931	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPE pPdpe, unsigned piPD)
3932	{
3933	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3934	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3935	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3936	if (pPml4e->n.u1Present)
3937	{
3938	PCX86PDPT pPdptTemp;
3939	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3940	AssertRCReturn(rc, NULL);
3941
3942	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3943	*pPdpe = pPdptTemp->a[iPdpt];
3944	if (pPdptTemp->a[iPdpt].n.u1Present)
3945	{
3946	PX86PDPAE pPD;
3947	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3948	AssertRCReturn(rc, NULL);
3949
3950	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3951	return pPD;
3952	}
3953	}
3954	return 0;
3955	}
3956
3957	#endif /* !IN_RC */
3958
3959	/**
3960	* Gets the shadow page directory, 32-bit.
3961	*
3962	* @returns Pointer to the shadow 32-bit PD.
3963	* @param pPGM Pointer to the PGM instance data.
3964	*/
3965	DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
3966	{
3967	return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
3968	}
3969
3970
3971	/**
3972	* Gets the shadow page directory entry for the specified address, 32-bit.
3973	*
3974	* @returns Shadow 32-bit PDE.
3975	* @param pPGM Pointer to the PGM instance data.
3976	* @param GCPtr The address.
3977	*/
3978	DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3979	{
3980	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
3981
3982	PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
3983	if (!pShwPde)
3984	{
3985	X86PDE ZeroPde = {0};
3986	return ZeroPde;
3987	}
3988	return pShwPde->a[iPd];
3989	}
3990
3991
3992	/**
3993	* Gets the pointer to the shadow page directory entry for the specified
3994	* address, 32-bit.
3995	*
3996	* @returns Pointer to the shadow 32-bit PDE.
3997	* @param pPGM Pointer to the PGM instance data.
3998	* @param GCPtr The address.
3999	*/
4000	DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4001	{
4002	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4003
4004	PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
4005	AssertReturn(pPde, NULL);
4006	return &pPde->a[iPd];
4007	}
4008
4009
4010	/**
4011	* Gets the shadow page pointer table, PAE.
4012	*
4013	* @returns Pointer to the shadow PAE PDPT.
4014	* @param pPGM Pointer to the PGM instance data.
4015	*/
4016	DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
4017	{
4018	return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4019	}
4020
4021
4022	/**
4023	* Gets the shadow page directory for the specified address, PAE.
4024	*
4025	* @returns Pointer to the shadow PD.
4026	* @param pPGM Pointer to the PGM instance data.
4027	* @param GCPtr The address.
4028	*/
4029	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4030	{
4031	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4032	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
4033
4034	if (!pPdpt->a[iPdpt].n.u1Present)
4035	return NULL;
4036
4037	/* Fetch the pgm pool shadow descriptor. */
4038	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(PGMCPU2VM(pPGM), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4039	AssertReturn(pShwPde, NULL);
4040
4041	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4042	}
4043
4044
4045	/**
4046	* Gets the shadow page directory for the specified address, PAE.
4047	*
4048	* @returns Pointer to the shadow PD.
4049	* @param pPGM Pointer to the PGM instance data.
4050	* @param GCPtr The address.
4051	*/
4052	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
4053	{
4054	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4055
4056	if (!pPdpt->a[iPdpt].n.u1Present)
4057	return NULL;
4058
4059	/* Fetch the pgm pool shadow descriptor. */
4060	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(PGMCPU2VM(pPGM), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4061	AssertReturn(pShwPde, NULL);
4062
4063	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4064	}
4065
4066
4067	/**
4068	* Gets the shadow page directory entry, PAE.
4069	*
4070	* @returns PDE.
4071	* @param pPGM Pointer to the PGM instance data.
4072	* @param GCPtr The address.
4073	*/
4074	DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4075	{
4076	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4077
4078	PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4079	if (!pShwPde)
4080	{
4081	X86PDEPAE ZeroPde = {0};
4082	return ZeroPde;
4083	}
4084	return pShwPde->a[iPd];
4085	}
4086
4087
4088	/**
4089	* Gets the pointer to the shadow page directory entry for an address, PAE.
4090	*
4091	* @returns Pointer to the PDE.
4092	* @param pPGM Pointer to the PGM instance data.
4093	* @param GCPtr The address.
4094	*/
4095	DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4096	{
4097	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4098
4099	PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4100	AssertReturn(pPde, NULL);
4101	return &pPde->a[iPd];
4102	}
4103
4104	#ifndef IN_RC
4105
4106	/**
4107	* Gets the shadow page map level-4 pointer.
4108	*
4109	* @returns Pointer to the shadow PML4.
4110	* @param pPGM Pointer to the PGM instance data.
4111	*/
4112	DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
4113	{
4114	return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4115	}
4116
4117
4118	/**
4119	* Gets the shadow page map level-4 entry for the specified address.
4120	*
4121	* @returns The entry.
4122	* @param pPGM Pointer to the PGM instance data.
4123	* @param GCPtr The address.
4124	*/
4125	DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
4126	{
4127	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4128	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4129
4130	if (!pShwPml4)
4131	{
4132	X86PML4E ZeroPml4e = {0};
4133	return ZeroPml4e;
4134	}
4135	return pShwPml4->a[iPml4];
4136	}
4137
4138
4139	/**
4140	* Gets the pointer to the specified shadow page map level-4 entry.
4141	*
4142	* @returns The entry.
4143	* @param pPGM Pointer to the PGM instance data.
4144	* @param iPml4 The PML4 index.
4145	*/
4146	DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4147	{
4148	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4149	if (!pShwPml4)
4150	return NULL;
4151	return &pShwPml4->a[iPml4];
4152	}
4153
4154
4155	/**
4156	* Gets the GUEST page directory pointer for the specified address.
4157	*
4158	* @returns The page directory in question.
4159	* @returns NULL if the page directory is not present or on an invalid page.
4160	* @param pPGM Pointer to the PGM instance data.
4161	* @param GCPtr The address.
4162	* @param piPD Receives the index into the returned page directory
4163	*/
4164	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
4165	{
4166	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4167	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4168	if (pGuestPml4->a[iPml4].n.u1Present)
4169	{
4170	PCX86PDPT pPdptTemp;
4171	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4172	AssertRCReturn(rc, NULL);
4173
4174	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4175	if (pPdptTemp->a[iPdpt].n.u1Present)
4176	{
4177	PX86PDPAE pPD;
4178	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4179	AssertRCReturn(rc, NULL);
4180
4181	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4182	return pPD;
4183	}
4184	}
4185	return NULL;
4186	}
4187
4188	#endif /* !IN_RC */
4189
4190	/**
4191	* Gets the page state for a physical handler.
4192	*
4193	* @returns The physical handler page state.
4194	* @param pCur The physical handler in question.
4195	*/
4196	DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
4197	{
4198	switch (pCur->enmType)
4199	{
4200	case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
4201	return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
4202
4203	case PGMPHYSHANDLERTYPE_MMIO:
4204	case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
4205	return PGM_PAGE_HNDL_PHYS_STATE_ALL;
4206
4207	default:
4208	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4209	}
4210	}
4211
4212
4213	/**
4214	* Gets the page state for a virtual handler.
4215	*
4216	* @returns The virtual handler page state.
4217	* @param pCur The virtual handler in question.
4218	* @remarks This should never be used on a hypervisor access handler.
4219	*/
4220	DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
4221	{
4222	switch (pCur->enmType)
4223	{
4224	case PGMVIRTHANDLERTYPE_WRITE:
4225	return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
4226	case PGMVIRTHANDLERTYPE_ALL:
4227	return PGM_PAGE_HNDL_VIRT_STATE_ALL;
4228	default:
4229	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4230	}
4231	}
4232
4233
4234	/**
4235	* Clears one physical page of a virtual handler
4236	*
4237	* @param pPGM Pointer to the PGM instance.
4238	* @param pCur Virtual handler structure
4239	* @param iPage Physical page index
4240	*
4241	* @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
4242	* need to care about other handlers in the same page.
4243	*/
4244	DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
4245	{
4246	const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
4247
4248	/*
4249	* Remove the node from the tree (it's supposed to be in the tree if we get here!).
4250	*/
4251	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4252	AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4253	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4254	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4255	#endif
4256	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
4257	{
4258	/* We're the head of the alias chain. */
4259	PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
4260	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4261	AssertReleaseMsg(pRemove != NULL,
4262	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4263	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4264	AssertReleaseMsg(pRemove == pPhys2Virt,
4265	("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4266	" got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4267	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
4268	pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
4269	#endif
4270	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4271	{
4272	/* Insert the next list in the alias chain into the tree. */
4273	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4274	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4275	AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4276	("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4277	pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
4278	#endif
4279	pNext->offNextAlias \|= PGMPHYS2VIRTHANDLER_IS_HEAD;
4280	bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
4281	AssertRelease(fRc);
4282	}
4283	}
4284	else
4285	{
4286	/* Locate the previous node in the alias chain. */
4287	PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
4288	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4289	AssertReleaseMsg(pPrev != pPhys2Virt,
4290	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4291	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4292	#endif
4293	for (;;)
4294	{
4295	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4296	if (pNext == pPhys2Virt)
4297	{
4298	/* unlink. */
4299	LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
4300	pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
4301	if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4302	pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
4303	else
4304	{
4305	PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4306	pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
4307	\| (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
4308	}
4309	break;
4310	}
4311
4312	/* next */
4313	if (pNext == pPrev)
4314	{
4315	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4316	AssertReleaseMsg(pNext != pPrev,
4317	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4318	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4319	#endif
4320	break;
4321	}
4322	pPrev = pNext;
4323	}
4324	}
4325	Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
4326	pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
4327	pPhys2Virt->offNextAlias = 0;
4328	pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
4329
4330	/*
4331	* Clear the ram flags for this page.
4332	*/
4333	PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
4334	AssertReturnVoid(pPage);
4335	PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
4336	}
4337
4338
4339	/**
4340	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4341	*
4342	* @returns Pointer to the shadow page structure.
4343	* @param pPool The pool.
4344	* @param HCPhys The HC physical address of the shadow page.
4345	*/
4346	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
4347	{
4348	/*
4349	* Look up the page.
4350	*/
4351	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
4352	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
4353	return pPage;
4354	}
4355
4356
4357	/**
4358	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4359	*
4360	* @returns Pointer to the shadow page structure.
4361	* @param pPool The pool.
4362	* @param idx The pool page index.
4363	*/
4364	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
4365	{
4366	AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
4367	return &pPool->aPages[idx];
4368	}
4369
4370
4371	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4372	/**
4373	* Clear references to guest physical memory.
4374	*
4375	* @param pPool The pool.
4376	* @param pPoolPage The pool page.
4377	* @param pPhysPage The physical guest page tracking structure.
4378	*/
4379	DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
4380	{
4381	/*
4382	* Just deal with the simple case here.
4383	*/
4384	# ifdef LOG_ENABLED
4385	const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
4386	# endif
4387	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4388	if (cRefs == 1)
4389	{
4390	Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
4391	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
4392	}
4393	else
4394	pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
4395	Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
4396	}
4397	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4398
4399
4400	#ifdef PGMPOOL_WITH_CACHE
4401	/**
4402	* Moves the page to the head of the age list.
4403	*
4404	* This is done when the cached page is used in one way or another.
4405	*
4406	* @param pPool The pool.
4407	* @param pPage The cached page.
4408	*/
4409	DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4410	{
4411	/*
4412	* Move to the head of the age list.
4413	*/
4414	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
4415	{
4416	/* unlink */
4417	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
4418	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
4419	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
4420	else
4421	pPool->iAgeTail = pPage->iAgePrev;
4422
4423	/* insert at head */
4424	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4425	pPage->iAgeNext = pPool->iAgeHead;
4426	Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
4427	pPool->iAgeHead = pPage->idx;
4428	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
4429	}
4430	}
4431	#endif /* PGMPOOL_WITH_CACHE */
4432
4433	/**
4434	* Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
4435	*
4436	* @param pVM VM Handle.
4437	* @param pPage PGM pool page
4438	*/
4439	DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4440	{
4441	Assert(!pPage->fLocked);
4442	pPage->fLocked = true;
4443	}
4444
4445
4446	/**
4447	* Unlocks a page to allow flushing again
4448	*
4449	* @param pVM VM Handle.
4450	* @param pPage PGM pool page
4451	*/
4452	DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4453	{
4454	Assert(pPage->fLocked);
4455	pPage->fLocked = false;
4456	}
4457
4458
4459	/**
4460	* Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
4461	*
4462	* @returns VBox status code.
4463	* @param pPage PGM pool page
4464	*/
4465	DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
4466	{
4467	if (pPage->fLocked)
4468	{
4469	LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
4470	if (pPage->cModifications)
4471	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
4472	return true;
4473	}
4474	return false;
4475	}
4476
4477	/**
4478	* Tells if mappings are to be put into the shadow page table or not
4479	*
4480	* @returns boolean result
4481	* @param pVM VM handle.
4482	*/
4483	DECLINLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
4484	{
4485	#ifdef IN_RING0
4486	/* There are no mappings in VT-x and AMD-V mode. */
4487	Assert(pPGM->fDisableMappings);
4488	return false;
4489	#else
4490	return !pPGM->fDisableMappings;
4491	#endif
4492	}
4493
4494	/** @} */
4495
4496	#endif
4497
4498

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMInternal.h@ 19167

Download in other formats: