PGMInternal.h@ 19488

Last change on this file since 19488 was 19488, checked in by vboxsync, 16 years ago
Made pgmPoolSyncCR3 a bit more SMP safe (pgmPoolClearAll still to be done).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 185.1 KB

Line
1	/* $Id: PGMInternal.h 19488 2009-05-07 13:54:51Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	#ifndef ___PGMInternal_h
23	#define ___PGMInternal_h
24
25	#include <VBox/cdefs.h>
26	#include <VBox/types.h>
27	#include <VBox/err.h>
28	#include <VBox/stam.h>
29	#include <VBox/param.h>
30	#include <VBox/vmm.h>
31	#include <VBox/mm.h>
32	#include <VBox/pdmcritsect.h>
33	#include <VBox/pdmapi.h>
34	#include <VBox/dis.h>
35	#include <VBox/dbgf.h>
36	#include <VBox/log.h>
37	#include <VBox/gmm.h>
38	#include <VBox/hwaccm.h>
39	#include <iprt/avl.h>
40	#include <iprt/assert.h>
41	#include <iprt/critsect.h>
42
43
44
45	/** @defgroup grp_pgm_int Internals
46	* @ingroup grp_pgm
47	* @internal
48	* @{
49	*/
50
51
52	/** @name PGM Compile Time Config
53	* @{
54	*/
55
56	/**
57	* Solve page is out of sync issues inside Guest Context (in PGMGC.cpp).
58	* Comment it if it will break something.
59	*/
60	#define PGM_OUT_OF_SYNC_IN_GC
61
62	/**
63	* Check and skip global PDEs for non-global flushes
64	*/
65	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
66
67	/**
68	* Sync N pages instead of a whole page table
69	*/
70	#define PGM_SYNC_N_PAGES
71
72	/**
73	* Number of pages to sync during a page fault
74	*
75	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
76	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
77	*/
78	#define PGM_SYNC_NR_PAGES 8
79
80	/**
81	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
82	*/
83	#define PGM_MAX_PHYSCACHE_ENTRIES 64
84	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
85
86	/**
87	* Enable caching of PGMR3PhysRead/WriteByte/Word/Dword
88	*/
89	#define PGM_PHYSMEMACCESS_CACHING
90
91	/** @def PGMPOOL_WITH_CACHE
92	* Enable agressive caching using the page pool.
93	*
94	* This requires PGMPOOL_WITH_USER_TRACKING and PGMPOOL_WITH_MONITORING.
95	*/
96	#define PGMPOOL_WITH_CACHE
97
98	/** @def PGMPOOL_WITH_MIXED_PT_CR3
99	* When defined, we'll deal with 'uncachable' pages.
100	*/
101	#ifdef PGMPOOL_WITH_CACHE
102	# define PGMPOOL_WITH_MIXED_PT_CR3
103	#endif
104
105	/** @def PGMPOOL_WITH_MONITORING
106	* Monitor the guest pages which are shadowed.
107	* When this is enabled, PGMPOOL_WITH_CACHE or PGMPOOL_WITH_GCPHYS_TRACKING must
108	* be enabled as well.
109	* @remark doesn't really work without caching now. (Mixed PT/CR3 change.)
110	*/
111	#ifdef PGMPOOL_WITH_CACHE
112	# define PGMPOOL_WITH_MONITORING
113	#endif
114
115	/** @def PGMPOOL_WITH_GCPHYS_TRACKING
116	* Tracking the of shadow pages mapping guest physical pages.
117	*
118	* This is very expensive, the current cache prototype is trying to figure out
119	* whether it will be acceptable with an agressive caching policy.
120	*/
121	#if defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
122	# define PGMPOOL_WITH_GCPHYS_TRACKING
123	#endif
124
125	/** @def PGMPOOL_WITH_USER_TRACKING
126	* Tracking users of shadow pages. This is required for the linking of shadow page
127	* tables and physical guest addresses.
128	*/
129	#if defined(PGMPOOL_WITH_GCPHYS_TRACKING) \|\| defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
130	# define PGMPOOL_WITH_USER_TRACKING
131	#endif
132
133	/** @def PGMPOOL_CFG_MAX_GROW
134	* The maximum number of pages to add to the pool in one go.
135	*/
136	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
137
138	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
139	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
140	*/
141	#ifdef VBOX_STRICT
142	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
143	#endif
144
145	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
146	* Enables the experimental lazy page allocation code. */
147	/# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
148
149	/** @} */
150
151
152	/** @name PDPT and PML4 flags.
153	* These are placed in the three bits available for system programs in
154	* the PDPT and PML4 entries.
155	* @{ */
156	/** The entry is a permanent one and it's must always be present.
157	* Never free such an entry. */
158	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
159	/** Mapping (hypervisor allocated pagetable). */
160	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
161	/** @} */
162
163	/** @name Page directory flags.
164	* These are placed in the three bits available for system programs in
165	* the page directory entries.
166	* @{ */
167	/** Mapping (hypervisor allocated pagetable). */
168	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
169	/** Made read-only to facilitate dirty bit tracking. */
170	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
171	/** @} */
172
173	/** @name Page flags.
174	* These are placed in the three bits available for system programs in
175	* the page entries.
176	* @{ */
177	/** Made read-only to facilitate dirty bit tracking. */
178	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
179
180	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
181	/** Scanned and approved by CSAM (tm).
182	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
183	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
184	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
185	#endif
186
187	/** @} */
188
189	/** @name Defines used to indicate the shadow and guest paging in the templates.
190	* @{ */
191	#define PGM_TYPE_REAL 1
192	#define PGM_TYPE_PROT 2
193	#define PGM_TYPE_32BIT 3
194	#define PGM_TYPE_PAE 4
195	#define PGM_TYPE_AMD64 5
196	#define PGM_TYPE_NESTED 6
197	#define PGM_TYPE_EPT 7
198	#define PGM_TYPE_MAX PGM_TYPE_EPT
199	/** @} */
200
201	/** Macro for checking if the guest is using paging.
202	* @param uGstType PGM_TYPE_*
203	* @param uShwType PGM_TYPE_*
204	* @remark ASSUMES certain order of the PGM_TYPE_* values.
205	*/
206	#define PGM_WITH_PAGING(uGstType, uShwType) \
207	( (uGstType) >= PGM_TYPE_32BIT \
208	&& (uShwType) != PGM_TYPE_NESTED \
209	&& (uShwType) != PGM_TYPE_EPT)
210
211	/** Macro for checking if the guest supports the NX bit.
212	* @param uGstType PGM_TYPE_*
213	* @param uShwType PGM_TYPE_*
214	* @remark ASSUMES certain order of the PGM_TYPE_* values.
215	*/
216	#define PGM_WITH_NX(uGstType, uShwType) \
217	( (uGstType) >= PGM_TYPE_PAE \
218	&& (uShwType) != PGM_TYPE_NESTED \
219	&& (uShwType) != PGM_TYPE_EPT)
220
221
222	/** @def PGM_HCPHYS_2_PTR
223	* Maps a HC physical page pool address to a virtual address.
224	*
225	* @returns VBox status code.
226	* @param pVM The VM handle.
227	* @param HCPhys The HC physical address to map to a virtual one.
228	* @param ppv Where to store the virtual address. No need to cast this.
229	*
230	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
231	* small page window employeed by that function. Be careful.
232	* @remark There is no need to assert on the result.
233	*/
234	#ifdef IN_RC
235	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
236	PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
237	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
238	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
239	pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
240	#else
241	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
242	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
243	#endif
244
245	/** @def PGM_HCPHYS_2_PTR_BY_PGM
246	* Maps a HC physical page pool address to a virtual address.
247	*
248	* @returns VBox status code.
249	* @param pPGM The PGM instance data.
250	* @param HCPhys The HC physical address to map to a virtual one.
251	* @param ppv Where to store the virtual address. No need to cast this.
252	*
253	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
254	* small page window employeed by that function. Be careful.
255	* @remark There is no need to assert on the result.
256	*/
257	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
258	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
259	pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
260	#else
261	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
262	PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
263	#endif
264
265	/** @def PGM_GCPHYS_2_PTR
266	* Maps a GC physical page address to a virtual address.
267	*
268	* @returns VBox status code.
269	* @param pVM The VM handle.
270	* @param GCPhys The GC physical address to map to a virtual one.
271	* @param ppv Where to store the virtual address. No need to cast this.
272	*
273	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
274	* small page window employeed by that function. Be careful.
275	* @remark There is no need to assert on the result.
276	*/
277	#ifdef IN_RC
278	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
279	PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
280	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
281	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
282	pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
283	#else
284	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
285	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
286	#endif
287
288	/** @def PGM_GCPHYS_2_PTR_BY_PGMCPU
289	* Maps a GC physical page address to a virtual address.
290	*
291	* @returns VBox status code.
292	* @param pPGM Pointer to the PGM instance data.
293	* @param GCPhys The GC physical address to map to a virtual one.
294	* @param ppv Where to store the virtual address. No need to cast this.
295	*
296	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
297	* small page window employeed by that function. Be careful.
298	* @remark There is no need to assert on the result.
299	*/
300	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
301	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
302	pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), GCPhys, (void **)(ppv))
303	#else
304	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
305	PGM_GCPHYS_2_PTR(PGMCPU2VM(pPGM), GCPhys, ppv)
306	#endif
307
308	/** @def PGM_GCPHYS_2_PTR_EX
309	* Maps a unaligned GC physical page address to a virtual address.
310	*
311	* @returns VBox status code.
312	* @param pVM The VM handle.
313	* @param GCPhys The GC physical address to map to a virtual one.
314	* @param ppv Where to store the virtual address. No need to cast this.
315	*
316	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
317	* small page window employeed by that function. Be careful.
318	* @remark There is no need to assert on the result.
319	*/
320	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
321	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
322	PGMDynMapGCPageOff(pVM, GCPhys, (void **)(ppv))
323	#else
324	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
325	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
326	#endif
327
328	/** @def PGM_INVL_PG
329	* Invalidates a page when in GC does nothing in HC.
330	*
331	* @param GCVirt The virtual address of the page to invalidate.
332	*/
333	#ifdef IN_RC
334	# define PGM_INVL_PG(GCVirt) ASMInvalidatePage((void *)(GCVirt))
335	#elif defined(IN_RING0)
336	# define PGM_INVL_PG(GCVirt) HWACCMInvalidatePage(pVM, (RTGCPTR)(GCVirt))
337	#else
338	# define PGM_INVL_PG(GCVirt) HWACCMInvalidatePage(pVM, (RTGCPTR)(GCVirt))
339	#endif
340
341	/** @def PGM_INVL_BIG_PG
342	* Invalidates a 4MB page directory entry when in GC does nothing in HC.
343	*
344	* @param GCVirt The virtual address within the page directory to invalidate.
345	*/
346	#ifdef IN_RC
347	# define PGM_INVL_BIG_PG(GCVirt) ASMReloadCR3()
348	#elif defined(IN_RING0)
349	# define PGM_INVL_BIG_PG(GCVirt) HWACCMFlushTLB(pVM)
350	#else
351	# define PGM_INVL_BIG_PG(GCVirt) HWACCMFlushTLB(pVM)
352	#endif
353
354	/** @def PGM_INVL_GUEST_TLBS()
355	* Invalidates all guest TLBs.
356	*/
357	#ifdef IN_RC
358	# define PGM_INVL_GUEST_TLBS() ASMReloadCR3()
359	#elif defined(IN_RING0)
360	# define PGM_INVL_GUEST_TLBS() HWACCMFlushTLB(pVM)
361	#else
362	# define PGM_INVL_GUEST_TLBS() HWACCMFlushTLB(pVM)
363	#endif
364
365	/** Size of the GCPtrConflict array in PGMMAPPING.
366	* @remarks Must be a power of two. */
367	#define PGMMAPPING_CONFLICT_MAX 8
368
369	/**
370	* Structure for tracking GC Mappings.
371	*
372	* This structure is used by linked list in both GC and HC.
373	*/
374	typedef struct PGMMAPPING
375	{
376	/** Pointer to next entry. */
377	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
378	/** Pointer to next entry. */
379	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
380	/** Pointer to next entry. */
381	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
382	/** Indicate whether this entry is finalized. */
383	bool fFinalized;
384	/** Start Virtual address. */
385	RTGCPTR GCPtr;
386	/** Last Virtual address (inclusive). */
387	RTGCPTR GCPtrLast;
388	/** Range size (bytes). */
389	RTGCPTR cb;
390	/** Pointer to relocation callback function. */
391	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
392	/** User argument to the callback. */
393	R3PTRTYPE(void *) pvUser;
394	/** Mapping description / name. For easing debugging. */
395	R3PTRTYPE(const char *) pszDesc;
396	/** Last 8 addresses that caused conflicts. */
397	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
398	/** Number of conflicts for this hypervisor mapping. */
399	uint32_t cConflicts;
400	/** Number of page tables. */
401	uint32_t cPTs;
402
403	/** Array of page table mapping data. Each entry
404	* describes one page table. The array can be longer
405	* than the declared length.
406	*/
407	struct
408	{
409	/** The HC physical address of the page table. */
410	RTHCPHYS HCPhysPT;
411	/** The HC physical address of the first PAE page table. */
412	RTHCPHYS HCPhysPaePT0;
413	/** The HC physical address of the second PAE page table. */
414	RTHCPHYS HCPhysPaePT1;
415	/** The HC virtual address of the 32-bit page table. */
416	R3PTRTYPE(PX86PT) pPTR3;
417	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
418	R3PTRTYPE(PX86PTPAE) paPaePTsR3;
419	/** The GC virtual address of the 32-bit page table. */
420	RCPTRTYPE(PX86PT) pPTRC;
421	/** The GC virtual address of the two PAE page table. */
422	RCPTRTYPE(PX86PTPAE) paPaePTsRC;
423	/** The GC virtual address of the 32-bit page table. */
424	R0PTRTYPE(PX86PT) pPTR0;
425	/** The GC virtual address of the two PAE page table. */
426	R0PTRTYPE(PX86PTPAE) paPaePTsR0;
427	} aPTs[1];
428	} PGMMAPPING;
429	/** Pointer to structure for tracking GC Mappings. */
430	typedef struct PGMMAPPING *PPGMMAPPING;
431
432
433	/**
434	* Physical page access handler structure.
435	*
436	* This is used to keep track of physical address ranges
437	* which are being monitored in some kind of way.
438	*/
439	typedef struct PGMPHYSHANDLER
440	{
441	AVLROGCPHYSNODECORE Core;
442	/** Access type. */
443	PGMPHYSHANDLERTYPE enmType;
444	/** Number of pages to update. */
445	uint32_t cPages;
446	/** Pointer to R3 callback function. */
447	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
448	/** User argument for R3 handlers. */
449	R3PTRTYPE(void *) pvUserR3;
450	/** Pointer to R0 callback function. */
451	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
452	/** User argument for R0 handlers. */
453	R0PTRTYPE(void *) pvUserR0;
454	/** Pointer to GC callback function. */
455	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
456	/** User argument for RC handlers. */
457	RCPTRTYPE(void *) pvUserRC;
458	/** Description / Name. For easing debugging. */
459	R3PTRTYPE(const char *) pszDesc;
460	#ifdef VBOX_WITH_STATISTICS
461	/** Profiling of this handler. */
462	STAMPROFILE Stat;
463	#endif
464	} PGMPHYSHANDLER;
465	/** Pointer to a physical page access handler structure. */
466	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
467
468
469	/**
470	* Cache node for the physical addresses covered by a virtual handler.
471	*/
472	typedef struct PGMPHYS2VIRTHANDLER
473	{
474	/** Core node for the tree based on physical ranges. */
475	AVLROGCPHYSNODECORE Core;
476	/** Offset from this struct to the PGMVIRTHANDLER structure. */
477	int32_t offVirtHandler;
478	/** Offset of the next alias relative to this one.
479	* Bit 0 is used for indicating whether we're in the tree.
480	* Bit 1 is used for indicating that we're the head node.
481	*/
482	int32_t offNextAlias;
483	} PGMPHYS2VIRTHANDLER;
484	/** Pointer to a phys to virtual handler structure. */
485	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
486
487	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
488	* node is in the tree. */
489	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
490	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
491	* node is in the head of an alias chain.
492	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
493	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
494	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
495	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
496
497
498	/**
499	* Virtual page access handler structure.
500	*
501	* This is used to keep track of virtual address ranges
502	* which are being monitored in some kind of way.
503	*/
504	typedef struct PGMVIRTHANDLER
505	{
506	/** Core node for the tree based on virtual ranges. */
507	AVLROGCPTRNODECORE Core;
508	/** Size of the range (in bytes). */
509	RTGCPTR cb;
510	/** Number of cache pages. */
511	uint32_t cPages;
512	/** Access type. */
513	PGMVIRTHANDLERTYPE enmType;
514	/** Pointer to the RC callback function. */
515	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
516	#if HC_ARCH_BITS == 64
517	RTRCPTR padding;
518	#endif
519	/** Pointer to the R3 callback function for invalidation. */
520	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
521	/** Pointer to the R3 callback function. */
522	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
523	/** Description / Name. For easing debugging. */
524	R3PTRTYPE(const char *) pszDesc;
525	#ifdef VBOX_WITH_STATISTICS
526	/** Profiling of this handler. */
527	STAMPROFILE Stat;
528	#endif
529	/** Array of cached physical addresses for the monitored ranged. */
530	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
531	} PGMVIRTHANDLER;
532	/** Pointer to a virtual page access handler structure. */
533	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
534
535
536	/**
537	* Page type.
538	*
539	* @remarks This enum has to fit in a 3-bit field (see PGMPAGE::u3Type).
540	* @remarks This is used in the saved state, so changes to it requires bumping
541	* the saved state version.
542	* @todo So, convert to \#defines!
543	*/
544	typedef enum PGMPAGETYPE
545	{
546	/** The usual invalid zero entry. */
547	PGMPAGETYPE_INVALID = 0,
548	/** RAM page. (RWX) */
549	PGMPAGETYPE_RAM,
550	/** MMIO2 page. (RWX) */
551	PGMPAGETYPE_MMIO2,
552	/** MMIO2 page aliased over an MMIO page. (RWX)
553	* See PGMHandlerPhysicalPageAlias(). */
554	PGMPAGETYPE_MMIO2_ALIAS_MMIO,
555	/** Shadowed ROM. (RWX) */
556	PGMPAGETYPE_ROM_SHADOW,
557	/** ROM page. (R-X) */
558	PGMPAGETYPE_ROM,
559	/** MMIO page. (---) */
560	PGMPAGETYPE_MMIO,
561	/** End of valid entries. */
562	PGMPAGETYPE_END
563	} PGMPAGETYPE;
564	AssertCompile(PGMPAGETYPE_END <= 7);
565
566	/** @name Page type predicates.
567	* @{ */
568	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
569	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
570	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
571	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
572	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
573	/** @} */
574
575
576	/**
577	* A Physical Guest Page tracking structure.
578	*
579	* The format of this structure is complicated because we have to fit a lot
580	* of information into as few bits as possible. The format is also subject
581	* to change (there is one comming up soon). Which means that for we'll be
582	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
583	* accessess to the structure.
584	*/
585	typedef struct PGMPAGE
586	{
587	/** The physical address and a whole lot of other stuff. All bits are used! */
588	RTHCPHYS HCPhysX;
589	/** The page state. */
590	uint32_t u2StateX : 2;
591	/** Flag indicating that a write monitored page was written to when set. */
592	uint32_t fWrittenToX : 1;
593	/** For later. */
594	uint32_t fSomethingElse : 1;
595	/** The Page ID.
596	* @todo Merge with HCPhysX once we've liberated HCPhysX of its stuff.
597	* The HCPhysX will then be 100% static. */
598	uint32_t idPageX : 28;
599	/** The page type (PGMPAGETYPE). */
600	uint32_t u3Type : 3;
601	/** The physical handler state (PGM_PAGE_HNDL_PHYS_STATE) /
602	uint32_t u2HandlerPhysStateX : 2;
603	/** The virtual handler state (PGM_PAGE_HNDL_VIRT_STATE) /
604	uint32_t u2HandlerVirtStateX : 2;
605	uint32_t u29B : 25;
606	} PGMPAGE;
607	AssertCompileSize(PGMPAGE, 16);
608	/** Pointer to a physical guest page. */
609	typedef PGMPAGE *PPGMPAGE;
610	/** Pointer to a const physical guest page. */
611	typedef const PGMPAGE *PCPGMPAGE;
612	/** Pointer to a physical guest page pointer. */
613	typedef PPGMPAGE *PPPGMPAGE;
614
615
616	/**
617	* Clears the page structure.
618	* @param pPage Pointer to the physical guest page tracking structure.
619	*/
620	#define PGM_PAGE_CLEAR(pPage) \
621	do { \
622	(pPage)->HCPhysX = 0; \
623	(pPage)->u2StateX = 0; \
624	(pPage)->fWrittenToX = 0; \
625	(pPage)->fSomethingElse = 0; \
626	(pPage)->idPageX = 0; \
627	(pPage)->u3Type = 0; \
628	(pPage)->u29B = 0; \
629	} while (0)
630
631	/**
632	* Initializes the page structure.
633	* @param pPage Pointer to the physical guest page tracking structure.
634	*/
635	#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
636	do { \
637	(pPage)->HCPhysX = (_HCPhys); \
638	(pPage)->u2StateX = (_uState); \
639	(pPage)->fWrittenToX = 0; \
640	(pPage)->fSomethingElse = 0; \
641	(pPage)->idPageX = (_idPage); \
642	/(pPage)->u3Type = (_uType); - later / \
643	PGM_PAGE_SET_TYPE(pPage, _uType); \
644	(pPage)->u29B = 0; \
645	} while (0)
646
647	/**
648	* Initializes the page structure of a ZERO page.
649	* @param pPage Pointer to the physical guest page tracking structure.
650	*/
651	#define PGM_PAGE_INIT_ZERO(pPage, pVM, _uType) \
652	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
653	/** Temporary hack. Replaced by PGM_PAGE_INIT_ZERO once the old code is kicked out. */
654	# define PGM_PAGE_INIT_ZERO_REAL(pPage, pVM, _uType) \
655	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
656
657
658	/** @name The Page state, PGMPAGE::u2StateX.
659	* @{ */
660	/** The zero page.
661	* This is a per-VM page that's never ever mapped writable. */
662	#define PGM_PAGE_STATE_ZERO 0
663	/** A allocated page.
664	* This is a per-VM page allocated from the page pool (or wherever
665	* we get MMIO2 pages from if the type is MMIO2).
666	*/
667	#define PGM_PAGE_STATE_ALLOCATED 1
668	/** A allocated page that's being monitored for writes.
669	* The shadow page table mappings are read-only. When a write occurs, the
670	* fWrittenTo member is set, the page remapped as read-write and the state
671	* moved back to allocated. */
672	#define PGM_PAGE_STATE_WRITE_MONITORED 2
673	/** The page is shared, aka. copy-on-write.
674	* This is a page that's shared with other VMs. */
675	#define PGM_PAGE_STATE_SHARED 3
676	/** @} */
677
678
679	/**
680	* Gets the page state.
681	* @returns page state (PGM_PAGE_STATE_*).
682	* @param pPage Pointer to the physical guest page tracking structure.
683	*/
684	#define PGM_PAGE_GET_STATE(pPage) ( (pPage)->u2StateX )
685
686	/**
687	* Sets the page state.
688	* @param pPage Pointer to the physical guest page tracking structure.
689	* @param _uState The new page state.
690	*/
691	#define PGM_PAGE_SET_STATE(pPage, _uState) \
692	do { (pPage)->u2StateX = (_uState); } while (0)
693
694
695	/**
696	* Gets the host physical address of the guest page.
697	* @returns host physical address (RTHCPHYS).
698	* @param pPage Pointer to the physical guest page tracking structure.
699	*/
700	#define PGM_PAGE_GET_HCPHYS(pPage) ( (pPage)->HCPhysX & UINT64_C(0x0000fffffffff000) )
701
702	/**
703	* Sets the host physical address of the guest page.
704	* @param pPage Pointer to the physical guest page tracking structure.
705	* @param _HCPhys The new host physical address.
706	*/
707	#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
708	do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0xffff000000000fff)) \
709	\| ((_HCPhys) & UINT64_C(0x0000fffffffff000)); } while (0)
710
711	/**
712	* Get the Page ID.
713	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
714	* @param pPage Pointer to the physical guest page tracking structure.
715	*/
716	#define PGM_PAGE_GET_PAGEID(pPage) ( (pPage)->idPageX )
717	/* later:
718	#define PGM_PAGE_GET_PAGEID(pPage) ( ((uint32_t)(pPage)->HCPhysX >> (48 - 12))
719	\| ((uint32_t)(pPage)->HCPhysX & 0xfff) )
720	*/
721	/**
722	* Sets the Page ID.
723	* @param pPage Pointer to the physical guest page tracking structure.
724	*/
725	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->idPageX = (_idPage); } while (0)
726	/* later:
727	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0x0000fffffffff000)) \
728	\| ((_idPage) & 0xfff) \
729	\| (((_idPage) & 0x0ffff000) << (48-12)); } while (0)
730	*/
731
732	/**
733	* Get the Chunk ID.
734	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
735	* @param pPage Pointer to the physical guest page tracking structure.
736	*/
737	#define PGM_PAGE_GET_CHUNKID(pPage) ( (pPage)->idPageX >> GMM_CHUNKID_SHIFT )
738	/* later:
739	#if GMM_CHUNKID_SHIFT == 12
740	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> 48) )
741	#elif GMM_CHUNKID_SHIFT > 12
742	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> (48 + (GMM_CHUNKID_SHIFT - 12)) )
743	#elif GMM_CHUNKID_SHIFT < 12
744	# define PGM_PAGE_GET_CHUNKID(pPage) ( ( (uint32_t)((pPage)->HCPhysX >> 48) << (12 - GMM_CHUNKID_SHIFT) ) \
745	\| ( (uint32_t)((pPage)->HCPhysX & 0xfff) >> GMM_CHUNKID_SHIFT ) )
746	#else
747	# error "GMM_CHUNKID_SHIFT isn't defined or something."
748	#endif
749	*/
750
751	/**
752	* Get the index of the page within the allocaiton chunk.
753	* @returns The page index.
754	* @param pPage Pointer to the physical guest page tracking structure.
755	*/
756	#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (pPage)->idPageX & GMM_PAGEID_IDX_MASK )
757	/* later:
758	#if GMM_CHUNKID_SHIFT <= 12
759	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & GMM_PAGEID_IDX_MASK) )
760	#else
761	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & 0xfff) \
762	\| ( (uint32_t)((pPage)->HCPhysX >> 48) & (RT_BIT_32(GMM_CHUNKID_SHIFT - 12) - 1) ) )
763	#endif
764	*/
765
766
767	/**
768	* Gets the page type.
769	* @returns The page type.
770	* @param pPage Pointer to the physical guest page tracking structure.
771	*/
772	#define PGM_PAGE_GET_TYPE(pPage) (pPage)->u3Type
773
774	/**
775	* Sets the page type.
776	* @param pPage Pointer to the physical guest page tracking structure.
777	* @param _enmType The new page type (PGMPAGETYPE).
778	*/
779	#define PGM_PAGE_SET_TYPE(pPage, _enmType) \
780	do { (pPage)->u3Type = (_enmType); } while (0)
781
782	/**
783	* Checks if the page is marked for MMIO.
784	* @returns true/false.
785	* @param pPage Pointer to the physical guest page tracking structure.
786	*/
787	#define PGM_PAGE_IS_MMIO(pPage) ( (pPage)->u3Type == PGMPAGETYPE_MMIO )
788
789	/**
790	* Checks if the page is backed by the ZERO page.
791	* @returns true/false.
792	* @param pPage Pointer to the physical guest page tracking structure.
793	*/
794	#define PGM_PAGE_IS_ZERO(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_ZERO )
795
796	/**
797	* Checks if the page is backed by a SHARED page.
798	* @returns true/false.
799	* @param pPage Pointer to the physical guest page tracking structure.
800	*/
801	#define PGM_PAGE_IS_SHARED(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_SHARED )
802
803
804	/**
805	* Marks the paget as written to (for GMM change monitoring).
806	* @param pPage Pointer to the physical guest page tracking structure.
807	*/
808	#define PGM_PAGE_SET_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 1; } while (0)
809
810	/**
811	* Clears the written-to indicator.
812	* @param pPage Pointer to the physical guest page tracking structure.
813	*/
814	#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 0; } while (0)
815
816	/**
817	* Checks if the page was marked as written-to.
818	* @returns true/false.
819	* @param pPage Pointer to the physical guest page tracking structure.
820	*/
821	#define PGM_PAGE_IS_WRITTEN_TO(pPage) ( (pPage)->fWrittenToX )
822
823
824	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateX).
825	*
826	* @remarks The values are assigned in order of priority, so we can calculate
827	* the correct state for a page with different handlers installed.
828	* @{ */
829	/** No handler installed. */
830	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
831	/** Monitoring is temporarily disabled. */
832	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
833	/** Write access is monitored. */
834	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
835	/** All access is monitored. */
836	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
837	/** @} */
838
839	/**
840	* Gets the physical access handler state of a page.
841	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
842	* @param pPage Pointer to the physical guest page tracking structure.
843	*/
844	#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) ( (pPage)->u2HandlerPhysStateX )
845
846	/**
847	* Sets the physical access handler state of a page.
848	* @param pPage Pointer to the physical guest page tracking structure.
849	* @param _uState The new state value.
850	*/
851	#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
852	do { (pPage)->u2HandlerPhysStateX = (_uState); } while (0)
853
854	/**
855	* Checks if the page has any physical access handlers, including temporariliy disabled ones.
856	* @returns true/false
857	* @param pPage Pointer to the physical guest page tracking structure.
858	*/
859	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE )
860
861	/**
862	* Checks if the page has any active physical access handlers.
863	* @returns true/false
864	* @param pPage Pointer to the physical guest page tracking structure.
865	*/
866	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
867
868
869	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateX).
870	*
871	* @remarks The values are assigned in order of priority, so we can calculate
872	* the correct state for a page with different handlers installed.
873	* @{ */
874	/** No handler installed. */
875	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
876	/* 1 is reserved so the lineup is identical with the physical ones. */
877	/** Write access is monitored. */
878	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
879	/** All access is monitored. */
880	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
881	/** @} */
882
883	/**
884	* Gets the virtual access handler state of a page.
885	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
886	* @param pPage Pointer to the physical guest page tracking structure.
887	*/
888	#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ( (pPage)->u2HandlerVirtStateX )
889
890	/**
891	* Sets the virtual access handler state of a page.
892	* @param pPage Pointer to the physical guest page tracking structure.
893	* @param _uState The new state value.
894	*/
895	#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
896	do { (pPage)->u2HandlerVirtStateX = (_uState); } while (0)
897
898	/**
899	* Checks if the page has any virtual access handlers.
900	* @returns true/false
901	* @param pPage Pointer to the physical guest page tracking structure.
902	*/
903	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ( (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
904
905	/**
906	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
907	* virtual handlers.
908	* @returns true/false
909	* @param pPage Pointer to the physical guest page tracking structure.
910	*/
911	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
912
913
914
915	/**
916	* Checks if the page has any access handlers, including temporarily disabled ones.
917	* @returns true/false
918	* @param pPage Pointer to the physical guest page tracking structure.
919	*/
920	#define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
921	( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE \
922	\|\| (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
923
924	/**
925	* Checks if the page has any active access handlers.
926	* @returns true/false
927	* @param pPage Pointer to the physical guest page tracking structure.
928	*/
929	#define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
930	( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
931	\|\| (pPage)->u2HandlerVirtStateX >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
932
933	/**
934	* Checks if the page has any active access handlers catching all accesses.
935	* @returns true/false
936	* @param pPage Pointer to the physical guest page tracking structure.
937	*/
938	#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
939	( (pPage)->u2HandlerPhysStateX == PGM_PAGE_HNDL_PHYS_STATE_ALL \
940	\|\| (pPage)->u2HandlerVirtStateX == PGM_PAGE_HNDL_VIRT_STATE_ALL )
941
942
943
944
945	/** @def PGM_PAGE_GET_TRACKING
946	* Gets the packed shadow page pool tracking data associated with a guest page.
947	* @returns uint16_t containing the data.
948	* @param pPage Pointer to the physical guest page tracking structure.
949	*/
950	#define PGM_PAGE_GET_TRACKING(pPage) \
951	( ((uint16_t )&(pPage)->HCPhysX + 3) )
952
953	/** @def PGM_PAGE_SET_TRACKING
954	* Sets the packed shadow page pool tracking data associated with a guest page.
955	* @param pPage Pointer to the physical guest page tracking structure.
956	* @param u16TrackingData The tracking data to store.
957	*/
958	#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
959	do { ((uint16_t )&(pPage)->HCPhysX + 3) = (u16TrackingData); } while (0)
960
961	/** @def PGM_PAGE_GET_TD_CREFS
962	* Gets the @a cRefs tracking data member.
963	* @returns cRefs.
964	* @param pPage Pointer to the physical guest page tracking structure.
965	*/
966	#define PGM_PAGE_GET_TD_CREFS(pPage) \
967	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
968
969	#define PGM_PAGE_GET_TD_IDX(pPage) \
970	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
971
972	/**
973	* Ram range for GC Phys to HC Phys conversion.
974	*
975	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
976	* conversions too, but we'll let MM handle that for now.
977	*
978	* This structure is used by linked lists in both GC and HC.
979	*/
980	typedef struct PGMRAMRANGE
981	{
982	/** Start of the range. Page aligned. */
983	RTGCPHYS GCPhys;
984	/** Size of the range. (Page aligned of course). */
985	RTGCPHYS cb;
986	/** Pointer to the next RAM range - for R3. */
987	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
988	/** Pointer to the next RAM range - for R0. */
989	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
990	/** Pointer to the next RAM range - for RC. */
991	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
992	/** PGM_RAM_RANGE_FLAGS_* flags. */
993	uint32_t fFlags;
994	/** Last address in the range (inclusive). Page aligned (-1). */
995	RTGCPHYS GCPhysLast;
996	/** Start of the HC mapping of the range. This is only used for MMIO2. */
997	R3PTRTYPE(void *) pvR3;
998	/** The range description. */
999	R3PTRTYPE(const char *) pszDesc;
1000	/** Pointer to self - R0 pointer. */
1001	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1002	/** Pointer to self - RC pointer. */
1003	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1004	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1005	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 1];
1006	/** Array of physical guest page tracking structures. */
1007	PGMPAGE aPages[1];
1008	} PGMRAMRANGE;
1009	/** Pointer to Ram range for GC Phys to HC Phys conversion. */
1010	typedef PGMRAMRANGE *PPGMRAMRANGE;
1011
1012	/** @name PGMRAMRANGE::fFlags
1013	* @{ */
1014	/** The RAM range is floating around as an independent guest mapping. */
1015	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1016	/** @} */
1017
1018
1019	/**
1020	* Per page tracking structure for ROM image.
1021	*
1022	* A ROM image may have a shadow page, in which case we may have
1023	* two pages backing it. This structure contains the PGMPAGE for
1024	* both while PGMRAMRANGE have a copy of the active one. It is
1025	* important that these aren't out of sync in any regard other
1026	* than page pool tracking data.
1027	*/
1028	typedef struct PGMROMPAGE
1029	{
1030	/** The page structure for the virgin ROM page. */
1031	PGMPAGE Virgin;
1032	/** The page structure for the shadow RAM page. */
1033	PGMPAGE Shadow;
1034	/** The current protection setting. */
1035	PGMROMPROT enmProt;
1036	/** Pad the structure size to a multiple of 8. */
1037	uint32_t u32Padding;
1038	} PGMROMPAGE;
1039	/** Pointer to a ROM page tracking structure. */
1040	typedef PGMROMPAGE *PPGMROMPAGE;
1041
1042
1043	/**
1044	* A registered ROM image.
1045	*
1046	* This is needed to keep track of ROM image since they generally
1047	* intrude into a PGMRAMRANGE. It also keeps track of additional
1048	* info like the two page sets (read-only virgin and read-write shadow),
1049	* the current state of each page.
1050	*
1051	* Because access handlers cannot easily be executed in a different
1052	* context, the ROM ranges needs to be accessible and in all contexts.
1053	*/
1054	typedef struct PGMROMRANGE
1055	{
1056	/** Pointer to the next range - R3. */
1057	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1058	/** Pointer to the next range - R0. */
1059	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1060	/** Pointer to the next range - RC. */
1061	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1062	/** Pointer alignment */
1063	RTRCPTR GCPtrAlignment;
1064	/** Address of the range. */
1065	RTGCPHYS GCPhys;
1066	/** Address of the last byte in the range. */
1067	RTGCPHYS GCPhysLast;
1068	/** Size of the range. */
1069	RTGCPHYS cb;
1070	/** The flags (PGMPHYS_ROM_FLAG_). /
1071	uint32_t fFlags;
1072	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1073	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 7 : 3];
1074	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1075	* This is used for strictness checks. */
1076	R3PTRTYPE(const void *) pvOriginal;
1077	/** The ROM description. */
1078	R3PTRTYPE(const char *) pszDesc;
1079	/** The per page tracking structures. */
1080	PGMROMPAGE aPages[1];
1081	} PGMROMRANGE;
1082	/** Pointer to a ROM range. */
1083	typedef PGMROMRANGE *PPGMROMRANGE;
1084
1085
1086	/**
1087	* A registered MMIO2 (= Device RAM) range.
1088	*
1089	* There are a few reason why we need to keep track of these
1090	* registrations. One of them is the deregistration & cleanup
1091	* stuff, while another is that the PGMRAMRANGE associated with
1092	* such a region may have to be removed from the ram range list.
1093	*
1094	* Overlapping with a RAM range has to be 100% or none at all. The
1095	* pages in the existing RAM range must not be ROM nor MMIO. A guru
1096	* meditation will be raised if a partial overlap or an overlap of
1097	* ROM pages is encountered. On an overlap we will free all the
1098	* existing RAM pages and put in the ram range pages instead.
1099	*/
1100	typedef struct PGMMMIO2RANGE
1101	{
1102	/** The owner of the range. (a device) */
1103	PPDMDEVINSR3 pDevInsR3;
1104	/** Pointer to the ring-3 mapping of the allocation. */
1105	RTR3PTR pvR3;
1106	/** Pointer to the next range - R3. */
1107	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1108	/** Whether it's mapped or not. */
1109	bool fMapped;
1110	/** Whether it's overlapping or not. */
1111	bool fOverlapping;
1112	/** The PCI region number.
1113	* @remarks This ASSUMES that nobody will ever really need to have multiple
1114	* PCI devices with matching MMIO region numbers on a single device. */
1115	uint8_t iRegion;
1116	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundrary. */
1117	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 1 : 5];
1118	/** The associated RAM range. */
1119	PGMRAMRANGE RamRange;
1120	} PGMMMIO2RANGE;
1121	/** Pointer to a MMIO2 range. */
1122	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1123
1124
1125
1126
1127	/**
1128	* PGMPhysRead/Write cache entry
1129	*/
1130	typedef struct PGMPHYSCACHEENTRY
1131	{
1132	/** R3 pointer to physical page. */
1133	R3PTRTYPE(uint8_t *) pbR3;
1134	/** GC Physical address for cache entry */
1135	RTGCPHYS GCPhys;
1136	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1137	RTGCPHYS u32Padding0; /*< alignment padding. /
1138	#endif
1139	} PGMPHYSCACHEENTRY;
1140
1141	/**
1142	* PGMPhysRead/Write cache to reduce REM memory access overhead
1143	*/
1144	typedef struct PGMPHYSCACHE
1145	{
1146	/** Bitmap of valid cache entries */
1147	uint64_t aEntries;
1148	/** Cache entries */
1149	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1150	} PGMPHYSCACHE;
1151
1152
1153	/** Pointer to an allocation chunk ring-3 mapping. */
1154	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1155	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1156	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1157
1158	/**
1159	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1160	*
1161	* The primary tree (Core) uses the chunk id as key.
1162	* The secondary tree (AgeCore) is used for ageing and uses ageing sequence number as key.
1163	*/
1164	typedef struct PGMCHUNKR3MAP
1165	{
1166	/** The key is the chunk id. */
1167	AVLU32NODECORE Core;
1168	/** The key is the ageing sequence number. */
1169	AVLLU32NODECORE AgeCore;
1170	/** The current age thingy. */
1171	uint32_t iAge;
1172	/** The current reference count. */
1173	uint32_t volatile cRefs;
1174	/** The current permanent reference count. */
1175	uint32_t volatile cPermRefs;
1176	/** The mapping address. */
1177	void *pv;
1178	} PGMCHUNKR3MAP;
1179
1180	/**
1181	* Allocation chunk ring-3 mapping TLB entry.
1182	*/
1183	typedef struct PGMCHUNKR3MAPTLBE
1184	{
1185	/** The chunk id. */
1186	uint32_t volatile idChunk;
1187	#if HC_ARCH_BITS == 64
1188	uint32_t u32Padding; /*< alignment padding. /
1189	#endif
1190	/** The chunk map. */
1191	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1192	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1193	#else
1194	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1195	#endif
1196	} PGMCHUNKR3MAPTLBE;
1197	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1198	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1199
1200	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1201	* @remark Must be a power of two value. */
1202	#define PGM_CHUNKR3MAPTLB_ENTRIES 32
1203
1204	/**
1205	* Allocation chunk ring-3 mapping TLB.
1206	*
1207	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1208	* At first glance this might look kinda odd since AVL trees are
1209	* supposed to give the most optimial lookup times of all trees
1210	* due to their balancing. However, take a tree with 1023 nodes
1211	* in it, that's 10 levels, meaning that most searches has to go
1212	* down 9 levels before they find what they want. This isn't fast
1213	* compared to a TLB hit. There is the factor of cache misses,
1214	* and of course the problem with trees and branch prediction.
1215	* This is why we use TLBs in front of most of the trees.
1216	*
1217	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1218	* difficult when we switch to the new inlined AVL trees (from kStuff).
1219	*/
1220	typedef struct PGMCHUNKR3MAPTLB
1221	{
1222	/** The TLB entries. */
1223	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1224	} PGMCHUNKR3MAPTLB;
1225
1226	/**
1227	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1228	* @returns Chunk TLB index.
1229	* @param idChunk The Chunk ID.
1230	*/
1231	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1232
1233
1234	/**
1235	* Ring-3 guest page mapping TLB entry.
1236	* @remarks used in ring-0 as well at the moment.
1237	*/
1238	typedef struct PGMPAGER3MAPTLBE
1239	{
1240	/** Address of the page. */
1241	RTGCPHYS volatile GCPhys;
1242	/** The guest page. */
1243	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1244	R3PTRTYPE(PPGMPAGE) volatile pPage;
1245	#else
1246	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1247	#endif
1248	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1249	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1250	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1251	#else
1252	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1253	#endif
1254	/** The address */
1255	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1256	R3PTRTYPE(void *) volatile pv;
1257	#else
1258	R3R0PTRTYPE(void *) volatile pv;
1259	#endif
1260	#if HC_ARCH_BITS == 32
1261	uint32_t u32Padding; /*< alignment padding. /
1262	#endif
1263	} PGMPAGER3MAPTLBE;
1264	/** Pointer to an entry in the HC physical TLB. */
1265	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1266
1267
1268	/** The number of entries in the ring-3 guest page mapping TLB.
1269	* @remarks The value must be a power of two. */
1270	#define PGM_PAGER3MAPTLB_ENTRIES 64
1271
1272	/**
1273	* Ring-3 guest page mapping TLB.
1274	* @remarks used in ring-0 as well at the moment.
1275	*/
1276	typedef struct PGMPAGER3MAPTLB
1277	{
1278	/** The TLB entries. */
1279	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1280	} PGMPAGER3MAPTLB;
1281	/** Pointer to the ring-3 guest page mapping TLB. */
1282	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1283
1284	/**
1285	* Calculates the index of the TLB entry for the specified guest page.
1286	* @returns Physical TLB index.
1287	* @param GCPhys The guest physical address.
1288	*/
1289	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1290
1291
1292	/**
1293	* Mapping cache usage set entry.
1294	*
1295	* @remarks 16-bit ints was choosen as the set is not expected to be used beyond
1296	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1297	* cache. If it's extended to include ring-3, well, then something will
1298	* have be changed here...
1299	*/
1300	typedef struct PGMMAPSETENTRY
1301	{
1302	/** The mapping cache index. */
1303	uint16_t iPage;
1304	/** The number of references.
1305	* The max is UINT16_MAX - 1. */
1306	uint16_t cRefs;
1307	#if HC_ARCH_BITS == 64
1308	uint32_t alignment;
1309	#endif
1310	/** Pointer to the page. */
1311	RTR0PTR pvPage;
1312	/** The physical address for this entry. */
1313	RTHCPHYS HCPhys;
1314	} PGMMAPSETENTRY;
1315	/** Pointer to a mapping cache usage set entry. */
1316	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1317
1318	/**
1319	* Mapping cache usage set.
1320	*
1321	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1322	* done during exits / traps. The set is
1323	*/
1324	typedef struct PGMMAPSET
1325	{
1326	/** The number of occupied entries.
1327	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1328	* dynamic mappings. */
1329	uint32_t cEntries;
1330	/** The start of the current subset.
1331	* This is UINT32_MAX if no subset is currently open. */
1332	uint32_t iSubset;
1333	/** The index of the current CPU, only valid if the set is open. */
1334	int32_t iCpu;
1335	#if HC_ARCH_BITS == 64
1336	uint32_t alignment;
1337	#endif
1338	/** The entries. */
1339	PGMMAPSETENTRY aEntries[64];
1340	/** HCPhys -> iEntry fast lookup table.
1341	* Use PGMMAPSET_HASH for hashing.
1342	* The entries may or may not be valid, check against cEntries. */
1343	uint8_t aiHashTable[128];
1344	} PGMMAPSET;
1345	/** Pointer to the mapping cache set. */
1346	typedef PGMMAPSET *PPGMMAPSET;
1347
1348	/** PGMMAPSET::cEntries value for a closed set. */
1349	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1350
1351	/** Hash function for aiHashTable. */
1352	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1353
1354	/** The max fill size (strict builds). */
1355	#define PGMMAPSET_MAX_FILL (64U * 80U / 100U)
1356
1357
1358	/** @name Context neutrual page mapper TLB.
1359	*
1360	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1361	* code is writting in a kind of context neutrual way. Time will show whether
1362	* this actually makes sense or not...
1363	*
1364	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1365	* context ends up using a global mapping cache on some platforms
1366	* (darwin).
1367	*
1368	* @{ */
1369	/** @typedef PPGMPAGEMAPTLB
1370	* The page mapper TLB pointer type for the current context. */
1371	/** @typedef PPGMPAGEMAPTLB
1372	* The page mapper TLB entry pointer type for the current context. */
1373	/** @typedef PPGMPAGEMAPTLB
1374	* The page mapper TLB entry pointer pointer type for the current context. */
1375	/** @def PGM_PAGEMAPTLB_ENTRIES
1376	* The number of TLB entries in the page mapper TLB for the current context. */
1377	/** @def PGM_PAGEMAPTLB_IDX
1378	* Calculate the TLB index for a guest physical address.
1379	* @returns The TLB index.
1380	* @param GCPhys The guest physical address. */
1381	/** @typedef PPGMPAGEMAP
1382	* Pointer to a page mapper unit for current context. */
1383	/** @typedef PPPGMPAGEMAP
1384	* Pointer to a page mapper unit pointer for current context. */
1385	#ifdef IN_RC
1386	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1387	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1388	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1389	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1390	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1391	typedef void * PPGMPAGEMAP;
1392	typedef void ** PPPGMPAGEMAP;
1393	//#elif IN_RING0
1394	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1395	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1396	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1397	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1398	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1399	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1400	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1401	#else
1402	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1403	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1404	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1405	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1406	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1407	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1408	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1409	#endif
1410	/** @} */
1411
1412
1413	/** @name PGM Pool Indexes.
1414	* Aka. the unique shadow page identifier.
1415	* @{ */
1416	/** NIL page pool IDX. */
1417	#define NIL_PGMPOOL_IDX 0
1418	/** The first normal index. */
1419	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1420	/** Page directory (32-bit root). */
1421	#define PGMPOOL_IDX_PD 1
1422	/** Page Directory Pointer Table (PAE root). */
1423	#define PGMPOOL_IDX_PDPT 2
1424	/** AMD64 CR3 level index.*/
1425	#define PGMPOOL_IDX_AMD64_CR3 3
1426	/** Nested paging root.*/
1427	#define PGMPOOL_IDX_NESTED_ROOT 4
1428	/** The first normal index. */
1429	#define PGMPOOL_IDX_FIRST 5
1430	/** The last valid index. (inclusive, 14 bits) */
1431	#define PGMPOOL_IDX_LAST 0x3fff
1432	/** @} */
1433
1434	/** The NIL index for the parent chain. */
1435	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1436
1437	/**
1438	* Node in the chain linking a shadowed page to it's parent (user).
1439	*/
1440	#pragma pack(1)
1441	typedef struct PGMPOOLUSER
1442	{
1443	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1444	uint16_t iNext;
1445	/** The user page index. */
1446	uint16_t iUser;
1447	/** Index into the user table. */
1448	uint32_t iUserTable;
1449	} PGMPOOLUSER, *PPGMPOOLUSER;
1450	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1451	#pragma pack()
1452
1453
1454	/** The NIL index for the phys ext chain. */
1455	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1456
1457	/**
1458	* Node in the chain of physical cross reference extents.
1459	* @todo Calling this an 'extent' is not quite right, find a better name.
1460	*/
1461	#pragma pack(1)
1462	typedef struct PGMPOOLPHYSEXT
1463	{
1464	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1465	uint16_t iNext;
1466	/** The user page index. */
1467	uint16_t aidx[3];
1468	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1469	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1470	#pragma pack()
1471
1472
1473	/**
1474	* The kind of page that's being shadowed.
1475	*/
1476	typedef enum PGMPOOLKIND
1477	{
1478	/** The virtual invalid 0 entry. */
1479	PGMPOOLKIND_INVALID = 0,
1480	/** The entry is free (=unused). */
1481	PGMPOOLKIND_FREE,
1482
1483	/** Shw: 32-bit page table; Gst: no paging */
1484	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1485	/** Shw: 32-bit page table; Gst: 32-bit page table. */
1486	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1487	/** Shw: 32-bit page table; Gst: 4MB page. */
1488	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1489	/** Shw: PAE page table; Gst: no paging */
1490	PGMPOOLKIND_PAE_PT_FOR_PHYS,
1491	/** Shw: PAE page table; Gst: 32-bit page table. */
1492	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1493	/** Shw: PAE page table; Gst: Half of a 4MB page. */
1494	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1495	/** Shw: PAE page table; Gst: PAE page table. */
1496	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1497	/** Shw: PAE page table; Gst: 2MB page. */
1498	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1499
1500	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1501	PGMPOOLKIND_32BIT_PD,
1502	/** Shw: 32-bit page directory. Gst: no paging. */
1503	PGMPOOLKIND_32BIT_PD_PHYS,
1504	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1505	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1506	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1507	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1508	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1509	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1510	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1511	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1512	/** Shw: PAE page directory; Gst: PAE page directory. */
1513	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1514	/** Shw: PAE page directory; Gst: no paging. */
1515	PGMPOOLKIND_PAE_PD_PHYS,
1516
1517	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1518	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1519	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1520	PGMPOOLKIND_PAE_PDPT,
1521	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1522	PGMPOOLKIND_PAE_PDPT_PHYS,
1523
1524	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1525	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1526	/** Shw: 64-bit page directory pointer table; Gst: no paging */
1527	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1528	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1529	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1530	/** Shw: 64-bit page directory table; Gst: no paging */
1531	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
1532
1533	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1534	PGMPOOLKIND_64BIT_PML4,
1535
1536	/** Shw: EPT page directory pointer table; Gst: no paging */
1537	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1538	/** Shw: EPT page directory table; Gst: no paging */
1539	PGMPOOLKIND_EPT_PD_FOR_PHYS,
1540	/** Shw: EPT page table; Gst: no paging */
1541	PGMPOOLKIND_EPT_PT_FOR_PHYS,
1542
1543	/** Shw: Root Nested paging table. */
1544	PGMPOOLKIND_ROOT_NESTED,
1545
1546	/** The last valid entry. */
1547	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1548	} PGMPOOLKIND;
1549
1550
1551	/**
1552	* The tracking data for a page in the pool.
1553	*/
1554	typedef struct PGMPOOLPAGE
1555	{
1556	/** AVL node code with the (R3) physical address of this page. */
1557	AVLOHCPHYSNODECORE Core;
1558	/** Pointer to the R3 mapping of the page. */
1559	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1560	R3PTRTYPE(void *) pvPageR3;
1561	#else
1562	R3R0PTRTYPE(void *) pvPageR3;
1563	#endif
1564	/** The guest physical address. */
1565	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
1566	uint32_t Alignment0;
1567	#endif
1568	RTGCPHYS GCPhys;
1569	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1570	uint8_t enmKind;
1571	uint8_t bPadding;
1572	/** The index of this page. */
1573	uint16_t idx;
1574	/** The next entry in the list this page currently resides in.
1575	* It's either in the free list or in the GCPhys hash. */
1576	uint16_t iNext;
1577	#ifdef PGMPOOL_WITH_USER_TRACKING
1578	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1579	uint16_t iUserHead;
1580	/** The number of present entries. */
1581	uint16_t cPresent;
1582	/** The first entry in the table which is present. */
1583	uint16_t iFirstPresent;
1584	#endif
1585	#ifdef PGMPOOL_WITH_MONITORING
1586	/** The number of modifications to the monitored page. */
1587	uint16_t cModifications;
1588	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
1589	uint16_t iModifiedNext;
1590	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
1591	uint16_t iModifiedPrev;
1592	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1593	uint16_t iMonitoredNext;
1594	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1595	uint16_t iMonitoredPrev;
1596	#endif
1597	#ifdef PGMPOOL_WITH_CACHE
1598	/** The next page in the age list. */
1599	uint16_t iAgeNext;
1600	/** The previous page in the age list. */
1601	uint16_t iAgePrev;
1602	#endif /* PGMPOOL_WITH_CACHE */
1603	/** Used to indicate that the page is zeroed. */
1604	bool fZeroed;
1605	/** Used to indicate that a PT has non-global entries. */
1606	bool fSeenNonGlobal;
1607	/** Used to indicate that we're monitoring writes to the guest page. */
1608	bool fMonitored;
1609	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1610	* (All pages are in the age list.) */
1611	bool fCached;
1612	/** This is used by the R3 access handlers when invoked by an async thread.
1613	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1614	bool volatile fReusedFlushPending;
1615	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
1616	bool fLocked;
1617	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
1618	/** Pointer to a const pool page. */
1619	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1620
1621
1622	#ifdef PGMPOOL_WITH_CACHE
1623	/** The hash table size. */
1624	# define PGMPOOL_HASH_SIZE 0x40
1625	/** The hash function. */
1626	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1627	#endif
1628
1629
1630	/**
1631	* The shadow page pool instance data.
1632	*
1633	* It's all one big allocation made at init time, except for the
1634	* pages that is. The user nodes follows immediatly after the
1635	* page structures.
1636	*/
1637	typedef struct PGMPOOL
1638	{
1639	/** The VM handle - R3 Ptr. */
1640	PVMR3 pVMR3;
1641	/** The VM handle - R0 Ptr. */
1642	PVMR0 pVMR0;
1643	/** The VM handle - RC Ptr. */
1644	PVMRC pVMRC;
1645	/** The max pool size. This includes the special IDs. */
1646	uint16_t cMaxPages;
1647	/** The current pool size. */
1648	uint16_t cCurPages;
1649	/** The head of the free page list. */
1650	uint16_t iFreeHead;
1651	/* Padding. */
1652	uint16_t u16Padding;
1653	#ifdef PGMPOOL_WITH_USER_TRACKING
1654	/** Head of the chain of free user nodes. */
1655	uint16_t iUserFreeHead;
1656	/** The number of user nodes we've allocated. */
1657	uint16_t cMaxUsers;
1658	/** The number of present page table entries in the entire pool. */
1659	uint32_t cPresent;
1660	/** Pointer to the array of user nodes - RC pointer. */
1661	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
1662	/** Pointer to the array of user nodes - R3 pointer. */
1663	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
1664	/** Pointer to the array of user nodes - R0 pointer. */
1665	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
1666	#endif /* PGMPOOL_WITH_USER_TRACKING */
1667	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1668	/** Head of the chain of free phys ext nodes. */
1669	uint16_t iPhysExtFreeHead;
1670	/** The number of user nodes we've allocated. */
1671	uint16_t cMaxPhysExts;
1672	/** Pointer to the array of physical xref extent - RC pointer. */
1673	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
1674	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
1675	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
1676	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
1677	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
1678	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1679	#ifdef PGMPOOL_WITH_CACHE
1680	/** Hash table for GCPhys addresses. */
1681	uint16_t aiHash[PGMPOOL_HASH_SIZE];
1682	/** The head of the age list. */
1683	uint16_t iAgeHead;
1684	/** The tail of the age list. */
1685	uint16_t iAgeTail;
1686	/** Set if the cache is enabled. */
1687	bool fCacheEnabled;
1688	#endif /* PGMPOOL_WITH_CACHE */
1689	#ifdef PGMPOOL_WITH_MONITORING
1690	/** Head of the list of modified pages. */
1691	uint16_t iModifiedHead;
1692	/** The current number of modified pages. */
1693	uint16_t cModifiedPages;
1694	/** Access handler, RC. */
1695	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
1696	/** Access handler, R0. */
1697	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
1698	/** Access handler, R3. */
1699	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
1700	/** The access handler description (HC ptr). */
1701	R3PTRTYPE(const char *) pszAccessHandler;
1702	#endif /* PGMPOOL_WITH_MONITORING */
1703	/** The number of pages currently in use. */
1704	uint16_t cUsedPages;
1705	#ifdef VBOX_WITH_STATISTICS
1706	/** The high wather mark for cUsedPages. */
1707	uint16_t cUsedPagesHigh;
1708	uint32_t Alignment1; /*< Align the next member on a 64-bit boundrary. /
1709	/** Profiling pgmPoolAlloc(). */
1710	STAMPROFILEADV StatAlloc;
1711	/** Profiling pgmPoolClearAll(). */
1712	STAMPROFILE StatClearAll;
1713	/** Profiling pgmPoolFlushAllInt(). */
1714	STAMPROFILE StatFlushAllInt;
1715	/** Profiling pgmPoolFlushPage(). */
1716	STAMPROFILE StatFlushPage;
1717	/** Profiling pgmPoolFree(). */
1718	STAMPROFILE StatFree;
1719	/** Profiling time spent zeroing pages. */
1720	STAMPROFILE StatZeroPage;
1721	# ifdef PGMPOOL_WITH_USER_TRACKING
1722	/** Profiling of pgmPoolTrackDeref. */
1723	STAMPROFILE StatTrackDeref;
1724	/** Profiling pgmTrackFlushGCPhysPT. */
1725	STAMPROFILE StatTrackFlushGCPhysPT;
1726	/** Profiling pgmTrackFlushGCPhysPTs. */
1727	STAMPROFILE StatTrackFlushGCPhysPTs;
1728	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
1729	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
1730	/** Number of times we've been out of user records. */
1731	STAMCOUNTER StatTrackFreeUpOneUser;
1732	# endif
1733	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1734	/** Profiling deref activity related tracking GC physical pages. */
1735	STAMPROFILE StatTrackDerefGCPhys;
1736	/** Number of linear searches for a HCPhys in the ram ranges. */
1737	STAMCOUNTER StatTrackLinearRamSearches;
1738	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
1739	STAMCOUNTER StamTrackPhysExtAllocFailures;
1740	# endif
1741	# ifdef PGMPOOL_WITH_MONITORING
1742	/** Profiling the RC/R0 access handler. */
1743	STAMPROFILE StatMonitorRZ;
1744	/** Times we've failed interpreting the instruction. */
1745	STAMCOUNTER StatMonitorRZEmulateInstr;
1746	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
1747	STAMPROFILE StatMonitorRZFlushPage;
1748	/** Times we've detected fork(). */
1749	STAMCOUNTER StatMonitorRZFork;
1750	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
1751	STAMPROFILE StatMonitorRZHandled;
1752	/** Times we've failed interpreting a patch code instruction. */
1753	STAMCOUNTER StatMonitorRZIntrFailPatch1;
1754	/** Times we've failed interpreting a patch code instruction during flushing. */
1755	STAMCOUNTER StatMonitorRZIntrFailPatch2;
1756	/** The number of times we've seen rep prefixes we can't handle. */
1757	STAMCOUNTER StatMonitorRZRepPrefix;
1758	/** Profiling the REP STOSD cases we've handled. */
1759	STAMPROFILE StatMonitorRZRepStosd;
1760
1761	/** Profiling the R3 access handler. */
1762	STAMPROFILE StatMonitorR3;
1763	/** Times we've failed interpreting the instruction. */
1764	STAMCOUNTER StatMonitorR3EmulateInstr;
1765	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
1766	STAMPROFILE StatMonitorR3FlushPage;
1767	/** Times we've detected fork(). */
1768	STAMCOUNTER StatMonitorR3Fork;
1769	/** Profiling the R3 access we've handled (except REP STOSD). */
1770	STAMPROFILE StatMonitorR3Handled;
1771	/** The number of times we've seen rep prefixes we can't handle. */
1772	STAMCOUNTER StatMonitorR3RepPrefix;
1773	/** Profiling the REP STOSD cases we've handled. */
1774	STAMPROFILE StatMonitorR3RepStosd;
1775	/** The number of times we're called in an async thread an need to flush. */
1776	STAMCOUNTER StatMonitorR3Async;
1777	/** The high wather mark for cModifiedPages. */
1778	uint16_t cModifiedPagesHigh;
1779	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundrary. /
1780	# endif
1781	# ifdef PGMPOOL_WITH_CACHE
1782	/** The number of cache hits. */
1783	STAMCOUNTER StatCacheHits;
1784	/** The number of cache misses. */
1785	STAMCOUNTER StatCacheMisses;
1786	/** The number of times we've got a conflict of 'kind' in the cache. */
1787	STAMCOUNTER StatCacheKindMismatches;
1788	/** Number of times we've been out of pages. */
1789	STAMCOUNTER StatCacheFreeUpOne;
1790	/** The number of cacheable allocations. */
1791	STAMCOUNTER StatCacheCacheable;
1792	/** The number of uncacheable allocations. */
1793	STAMCOUNTER StatCacheUncacheable;
1794	# endif
1795	#elif HC_ARCH_BITS == 64
1796	uint32_t Alignment3; /*< Align the next member on a 64-bit boundrary. /
1797	#endif
1798	/** The AVL tree for looking up a page by its HC physical address. */
1799	AVLOHCPHYSTREE HCPhysTree;
1800	uint32_t Alignment4; /*< Align the next member on a 64-bit boundrary. /
1801	/** Array of pages. (cMaxPages in length)
1802	* The Id is the index into thist array.
1803	*/
1804	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
1805	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
1806
1807
1808	/** @def PGMPOOL_PAGE_2_PTR
1809	* Maps a pool page pool into the current context.
1810	*
1811	* @returns VBox status code.
1812	* @param pVM The VM handle.
1813	* @param pPage The pool page.
1814	*
1815	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1816	* small page window employeed by that function. Be careful.
1817	* @remark There is no need to assert on the result.
1818	*/
1819	#if defined(IN_RC)
1820	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1821	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1822	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1823	#elif defined(VBOX_STRICT)
1824	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageStrict(pPage)
1825	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
1826	{
1827	Assert(pPage && pPage->pvPageR3);
1828	return pPage->pvPageR3;
1829	}
1830	#else
1831	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) ((pPage)->pvPageR3)
1832	#endif
1833
1834	/** @def PGMPOOL_PAGE_2_PTR_BY_PGM
1835	* Maps a pool page pool into the current context.
1836	*
1837	* @returns VBox status code.
1838	* @param pPGM Pointer to the PGM instance data.
1839	* @param pPage The pool page.
1840	*
1841	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1842	* small page window employeed by that function. Be careful.
1843	* @remark There is no need to assert on the result.
1844	*/
1845	#if defined(IN_RC)
1846	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1847	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1848	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1849	#else
1850	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGM2VM(pPGM), pPage)
1851	#endif
1852
1853	/** @def PGMPOOL_PAGE_2_PTR_BY_PGMCPU
1854	* Maps a pool page pool into the current context.
1855	*
1856	* @returns VBox status code.
1857	* @param pPGM Pointer to the PGMCPU instance data.
1858	* @param pPage The pool page.
1859	*
1860	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1861	* small page window employeed by that function. Be careful.
1862	* @remark There is no need to assert on the result.
1863	*/
1864	#if defined(IN_RC)
1865	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1866	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1867	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1868	#else
1869	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGMCPU2VM(pPGM), pPage)
1870	#endif
1871
1872
1873	/** @name Per guest page tracking data.
1874	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
1875	* is to use more bits for it and split it up later on. But for now we'll play
1876	* safe and change as little as possible.
1877	*
1878	* The 16-bit word has two parts:
1879	*
1880	* The first 14-bit forms the @a idx field. It is either the index of a page in
1881	* the shadow page pool, or and index into the extent list.
1882	*
1883	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
1884	* shadow page pool references to the page. If cRefs equals
1885	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
1886	* (misnomer) table and not the shadow page pool.
1887	*
1888	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
1889	* the 16-bit word.
1890	*
1891	* @{ */
1892	/** The shift count for getting to the cRefs part. */
1893	#define PGMPOOL_TD_CREFS_SHIFT 14
1894	/** The mask applied after shifting the tracking data down by
1895	* PGMPOOL_TD_CREFS_SHIFT. */
1896	#define PGMPOOL_TD_CREFS_MASK 0x3
1897	/** The cRef value used to indiciate that the idx is the head of a
1898	* physical cross reference list. */
1899	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
1900	/** The shift used to get idx. */
1901	#define PGMPOOL_TD_IDX_SHIFT 0
1902	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
1903	#define PGMPOOL_TD_IDX_MASK 0x3fff
1904	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
1905	* simply too many mappings of this page. */
1906	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
1907
1908	/** @def PGMPOOL_TD_MAKE
1909	* Makes a 16-bit tracking data word.
1910	*
1911	* @returns tracking data.
1912	* @param cRefs The @a cRefs field. Must be within bounds!
1913	* @param idx The @a idx field. Must also be within bounds! */
1914	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
1915
1916	/** @def PGMPOOL_TD_GET_CREFS
1917	* Get the @a cRefs field from a tracking data word.
1918	*
1919	* @returns The @a cRefs field
1920	* @param u16 The tracking data word. */
1921	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
1922
1923	/** @def PGMPOOL_TD_GET_IDX
1924	* Get the @a idx field from a tracking data word.
1925	*
1926	* @returns The @a idx field
1927	* @param u16 The tracking data word. */
1928	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
1929	/** @} */
1930
1931
1932	/**
1933	* Trees are using self relative offsets as pointers.
1934	* So, all its data, including the root pointer, must be in the heap for HC and GC
1935	* to have the same layout.
1936	*/
1937	typedef struct PGMTREES
1938	{
1939	/** Physical access handlers (AVL range+offsetptr tree). */
1940	AVLROGCPHYSTREE PhysHandlers;
1941	/** Virtual access handlers (AVL range + GC ptr tree). */
1942	AVLROGCPTRTREE VirtHandlers;
1943	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
1944	AVLROGCPHYSTREE PhysToVirtHandlers;
1945	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
1946	AVLROGCPTRTREE HyperVirtHandlers;
1947	} PGMTREES;
1948	/** Pointer to PGM trees. */
1949	typedef PGMTREES *PPGMTREES;
1950
1951
1952	/** @name Paging mode macros
1953	* @{ */
1954	#ifdef IN_RC
1955	# define PGM_CTX(a,b) a##RC##b
1956	# define PGM_CTX_STR(a,b) a "GC" b
1957	# define PGM_CTX_DECL(type) VMMRCDECL(type)
1958	#else
1959	# ifdef IN_RING3
1960	# define PGM_CTX(a,b) a##R3##b
1961	# define PGM_CTX_STR(a,b) a "R3" b
1962	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
1963	# else
1964	# define PGM_CTX(a,b) a##R0##b
1965	# define PGM_CTX_STR(a,b) a "R0" b
1966	# define PGM_CTX_DECL(type) VMMDECL(type)
1967	# endif
1968	#endif
1969
1970	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
1971	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
1972	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
1973	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
1974	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
1975	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
1976	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
1977	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
1978	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
1979	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
1980	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
1981	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
1982	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
1983	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
1984	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
1985	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
1986	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
1987
1988	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
1989	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
1990	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
1991	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
1992	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
1993	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
1994	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
1995	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
1996	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
1997	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
1998	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
1999	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2000	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2001	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2002	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2003	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2004	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2005
2006	/* Shw_Gst */
2007	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2008	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2009	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2010	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2011	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2012	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2013	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2014	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2015	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2016	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2017	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2018	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2019	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2020	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2021	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2022	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2023	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2024	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2025	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2026
2027	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2028	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2029	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2030	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2031	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2032	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2033	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2034	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2035	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2036	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2037	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2038	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2039	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2040	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2041	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2042	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2043	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2044	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2045	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2046	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2047	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2048	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2049	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2050	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2051	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2052	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2053	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2054	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2055	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2056	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2057	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2058	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2059	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2060	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2061	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2062	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2063	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2064
2065	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2066	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2067	/** @} */
2068
2069	/**
2070	* Data for each paging mode.
2071	*/
2072	typedef struct PGMMODEDATA
2073	{
2074	/** The guest mode type. */
2075	uint32_t uGstType;
2076	/** The shadow mode type. */
2077	uint32_t uShwType;
2078
2079	/** @name Function pointers for Shadow paging.
2080	* @{
2081	*/
2082	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2083	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2084	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2085	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2086
2087	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2088	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2089
2090	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2091	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2092	/** @} */
2093
2094	/** @name Function pointers for Guest paging.
2095	* @{
2096	*/
2097	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2098	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2099	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2100	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2101	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2102	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2103	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2104	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2105	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2106	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2107	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2108	/** @} */
2109
2110	/** @name Function pointers for Both Shadow and Guest paging.
2111	* @{
2112	*/
2113	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2114	/* no pfnR3BthTrap0eHandler */
2115	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2116	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2117	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2118	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2119	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2120	#ifdef VBOX_STRICT
2121	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2122	#endif
2123	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2124	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2125
2126	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2127	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2128	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2129	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2130	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2131	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2132	#ifdef VBOX_STRICT
2133	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2134	#endif
2135	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2136	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2137
2138	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2139	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2140	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2141	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2142	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2143	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2144	#ifdef VBOX_STRICT
2145	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2146	#endif
2147	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2148	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2149	/** @} */
2150	} PGMMODEDATA, *PPGMMODEDATA;
2151
2152
2153
2154	/**
2155	* Converts a PGM pointer into a VM pointer.
2156	* @returns Pointer to the VM structure the PGM is part of.
2157	* @param pPGM Pointer to PGM instance data.
2158	*/
2159	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2160
2161	/**
2162	* PGM Data (part of VM)
2163	*/
2164	typedef struct PGM
2165	{
2166	/** Offset to the VM structure. */
2167	RTINT offVM;
2168	/** Offset of the PGMCPU structure relative to VMCPU. */
2169	RTINT offVCpuPGM;
2170
2171	/** @cfgm{RamPreAlloc, boolean, false}
2172	* Indicates whether the base RAM should all be allocated before starting
2173	* the VM (default), or if it should be allocated when first written to.
2174	*/
2175	bool fRamPreAlloc;
2176	/** Alignment padding. */
2177	bool afAlignment0[7];
2178
2179	/** What needs syncing (PGM_SYNC_*).
2180	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2181	* PGMFlushTLB, and PGMR3Load. */
2182	RTUINT fGlobalSyncFlags;
2183
2184	/*
2185	* This will be redefined at least two more times before we're done, I'm sure.
2186	* The current code is only to get on with the coding.
2187	* - 2004-06-10: initial version, bird.
2188	* - 2004-07-02: 1st time, bird.
2189	* - 2004-10-18: 2nd time, bird.
2190	* - 2005-07-xx: 3rd time, bird.
2191	*/
2192
2193	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2194	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
2195	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2196	RCPTRTYPE(PX86PTEPAE) paDynPageMapPaePTEsGC;
2197
2198	/** The host paging mode. (This is what SUPLib reports.) */
2199	SUPPAGINGMODE enmHostMode;
2200
2201	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2202	RTGCPHYS GCPhys4MBPSEMask;
2203
2204	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2205	* This is sorted by physical address and contains no overlapping ranges. */
2206	R3PTRTYPE(PPGMRAMRANGE) pRamRangesR3;
2207	/** R0 pointer corresponding to PGM::pRamRangesR3. */
2208	R0PTRTYPE(PPGMRAMRANGE) pRamRangesR0;
2209	/** RC pointer corresponding to PGM::pRamRangesR3. */
2210	RCPTRTYPE(PPGMRAMRANGE) pRamRangesRC;
2211	RTRCPTR alignment4; /*< structure alignment. /
2212
2213	/** Pointer to the list of ROM ranges - for R3.
2214	* This is sorted by physical address and contains no overlapping ranges. */
2215	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2216	/** R0 pointer corresponding to PGM::pRomRangesR3. */
2217	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2218	/** RC pointer corresponding to PGM::pRomRangesR3. */
2219	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
2220	/** Alignment padding. */
2221	RTRCPTR GCPtrPadding2;
2222
2223	/** Pointer to the list of MMIO2 ranges - for R3.
2224	* Registration order. */
2225	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
2226
2227	/** PGM offset based trees - R3 Ptr. */
2228	R3PTRTYPE(PPGMTREES) pTreesR3;
2229	/** PGM offset based trees - R0 Ptr. */
2230	R0PTRTYPE(PPGMTREES) pTreesR0;
2231	/** PGM offset based trees - RC Ptr. */
2232	RCPTRTYPE(PPGMTREES) pTreesRC;
2233
2234	/** Linked list of GC mappings - for RC.
2235	* The list is sorted ascending on address.
2236	*/
2237	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
2238	/** Linked list of GC mappings - for HC.
2239	* The list is sorted ascending on address.
2240	*/
2241	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
2242	/** Linked list of GC mappings - for R0.
2243	* The list is sorted ascending on address.
2244	*/
2245	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
2246
2247	/** Pointer to the 5 page CR3 content mapping.
2248	* The first page is always the CR3 (in some form) while the 4 other pages
2249	* are used of the PDs in PAE mode. */
2250	RTGCPTR GCPtrCR3Mapping;
2251	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
2252	uint32_t u32Alignment;
2253	#endif
2254
2255	/** Indicates that PGMR3FinalizeMappings has been called and that further
2256	* PGMR3MapIntermediate calls will be rejected. */
2257	bool fFinalizedMappings;
2258	/** If set no conflict checks are required. (boolean) */
2259	bool fMappingsFixed;
2260	/** If set, then no mappings are put into the shadow page table. (boolean) */
2261	bool fDisableMappings;
2262	/** Size of fixed mapping */
2263	uint32_t cbMappingFixed;
2264	/** Base address (GC) of fixed mapping */
2265	RTGCPTR GCPtrMappingFixed;
2266	/** The address of the previous RAM range mapping. */
2267	RTGCPTR GCPtrPrevRamRangeMapping;
2268
2269	/** @name Intermediate Context
2270	* @{ */
2271	/** Pointer to the intermediate page directory - Normal. */
2272	R3PTRTYPE(PX86PD) pInterPD;
2273	/** Pointer to the intermedate page tables - Normal.
2274	* There are two page tables, one for the identity mapping and one for
2275	* the host context mapping (of the core code). */
2276	R3PTRTYPE(PX86PT) apInterPTs[2];
2277	/** Pointer to the intermedate page tables - PAE. */
2278	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
2279	/** Pointer to the intermedate page directory - PAE. */
2280	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
2281	/** Pointer to the intermedate page directory - PAE. */
2282	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
2283	/** Pointer to the intermedate page-map level 4 - AMD64. */
2284	R3PTRTYPE(PX86PML4) pInterPaePML4;
2285	/** Pointer to the intermedate page directory - AMD64. */
2286	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
2287	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
2288	RTHCPHYS HCPhysInterPD;
2289	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
2290	RTHCPHYS HCPhysInterPaePDPT;
2291	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
2292	RTHCPHYS HCPhysInterPaePML4;
2293	/** @} */
2294
2295	/** Base address of the dynamic page mapping area.
2296	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
2297	*/
2298	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
2299	/** The index of the last entry used in the dynamic page mapping area. */
2300	RTUINT iDynPageMapLast;
2301	/** Cache containing the last entries in the dynamic page mapping area.
2302	* The cache size is covering half of the mapping area. */
2303	RTHCPHYS aHCPhysDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT + 1)];
2304	/** Keep a lock counter for the full (!) mapping area. */
2305	uint32_t aLockedDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT)];
2306
2307	/** The address of the ring-0 mapping cache if we're making use of it. */
2308	RTR0PTR pvR0DynMapUsed;
2309	#if HC_ARCH_BITS == 32
2310	RTR0PTR R0PtrPadding0; /*< Alignment. /
2311	#endif
2312
2313	/** PGM critical section.
2314	* This protects the physical & virtual access handlers, ram ranges,
2315	* and the page flag updating (some of it anyway).
2316	*/
2317	PDMCRITSECT CritSect;
2318
2319	/** Pointer to SHW+GST mode data (function pointers).
2320	* The index into this table is made up from */
2321	R3PTRTYPE(PPGMMODEDATA) paModeData;
2322
2323	/** Shadow Page Pool - R3 Ptr. */
2324	R3PTRTYPE(PPGMPOOL) pPoolR3;
2325	/** Shadow Page Pool - R0 Ptr. */
2326	R0PTRTYPE(PPGMPOOL) pPoolR0;
2327	/** Shadow Page Pool - RC Ptr. */
2328	RCPTRTYPE(PPGMPOOL) pPoolRC;
2329
2330	/** We're not in a state which permits writes to guest memory.
2331	* (Only used in strict builds.) */
2332	bool fNoMorePhysWrites;
2333
2334	/** Flush the cache on the next access. */
2335	bool fPhysCacheFlushPending;
2336	/** @todo r=bird: Fix member names!*/
2337	/** PGMPhysRead cache */
2338	PGMPHYSCACHE pgmphysreadcache;
2339	/** PGMPhysWrite cache */
2340	PGMPHYSCACHE pgmphyswritecache;
2341
2342	/**
2343	* Data associated with managing the ring-3 mappings of the allocation chunks.
2344	*/
2345	struct
2346	{
2347	/** The chunk tree, ordered by chunk id. */
2348	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2349	R3PTRTYPE(PAVLU32NODECORE) pTree;
2350	#else
2351	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
2352	#endif
2353	/** The chunk mapping TLB. */
2354	PGMCHUNKR3MAPTLB Tlb;
2355	/** The number of mapped chunks. */
2356	uint32_t c;
2357	/** The maximum number of mapped chunks.
2358	* @cfgm PGM/MaxRing3Chunks */
2359	uint32_t cMax;
2360	/** The chunk age tree, ordered by ageing sequence number. */
2361	R3PTRTYPE(PAVLLU32NODECORE) pAgeTree;
2362	/** The current time. */
2363	uint32_t iNow;
2364	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
2365	uint32_t AgeingCountdown;
2366	} ChunkR3Map;
2367
2368	/**
2369	* The page mapping TLB for ring-3 and (for the time being) ring-0.
2370	*/
2371	PGMPAGER3MAPTLB PhysTlbHC;
2372
2373	/** @name The zero page.
2374	* @{ */
2375	/** The host physical address of the zero page. */
2376	RTHCPHYS HCPhysZeroPg;
2377	/** The ring-3 mapping of the zero page. */
2378	RTR3PTR pvZeroPgR3;
2379	/** The ring-0 mapping of the zero page. */
2380	RTR0PTR pvZeroPgR0;
2381	/** The GC mapping of the zero page. */
2382	RTGCPTR pvZeroPgRC;
2383	#if GC_ARCH_BITS != 32
2384	uint32_t u32ZeroAlignment; /*< Alignment padding. /
2385	#endif
2386	/** @}*/
2387
2388	/** The number of handy pages. */
2389	uint32_t cHandyPages;
2390	/**
2391	* Array of handy pages.
2392	*
2393	* This array is used in a two way communication between pgmPhysAllocPage
2394	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
2395	* an intermediary.
2396	*
2397	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
2398	* (The current size of 32 pages, means 128 KB of handy memory.)
2399	*/
2400	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
2401
2402	/** @name Error injection.
2403	* @{ */
2404	/** Inject handy page allocation errors pretending we're completely out of
2405	* memory. */
2406	bool volatile fErrInjHandyPages;
2407	/** Padding. */
2408	bool afReserved[7];
2409	/** @} */
2410
2411	/** @name Release Statistics
2412	* @{ */
2413	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero.) /
2414	uint32_t cPrivatePages; /*< The number of private pages. /
2415	uint32_t cSharedPages; /*< The number of shared pages. /
2416	uint32_t cZeroPages; /*< The number of zero backed pages. /
2417
2418	/** The number of times we were forced to change the hypervisor region location. */
2419	STAMCOUNTER cRelocations;
2420	/** @} */
2421
2422	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2423	/* R3 only: */
2424	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2425	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2426
2427	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2428	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2429	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2430	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2431	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2432	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2433	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2434	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2435	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2436	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2437	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2438	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2439	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2440	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2441	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2442	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2443	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2444	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2445	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2446	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2447	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2448	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2449
2450	/* RC only: */
2451	STAMCOUNTER StatRCDynMapCacheMisses; /*< RC: The number of dynamic page mapping cache misses /
2452	STAMCOUNTER StatRCDynMapCacheHits; /*< RC: The number of dynamic page mapping cache hits /
2453	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2454	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2455
2456	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2457	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2458	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2459	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2460	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2461	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2462	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2463	# endif
2464	#endif
2465	} PGM;
2466	/** Pointer to the PGM instance data. */
2467	typedef PGM *PPGM;
2468
2469
2470	/**
2471	* Converts a PGMCPU pointer into a VM pointer.
2472	* @returns Pointer to the VM structure the PGM is part of.
2473	* @param pPGM Pointer to PGMCPU instance data.
2474	*/
2475	#define PGMCPU2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2476
2477	/**
2478	* Converts a PGMCPU pointer into a PGM pointer.
2479	* @returns Pointer to the VM structure the PGM is part of.
2480	* @param pPGM Pointer to PGMCPU instance data.
2481	*/
2482	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char*)pPGMCpu - pPGMCpu->offPGM) )
2483
2484	/**
2485	* PGMCPU Data (part of VMCPU).
2486	*/
2487	typedef struct PGMCPU
2488	{
2489	/** Offset to the VM structure. */
2490	RTINT offVM;
2491	/** Offset to the VMCPU structure. */
2492	RTINT offVCpu;
2493	/** Offset of the PGM structure relative to VMCPU. */
2494	RTINT offPGM;
2495	RTINT uPadding0; /*< structure size alignment. /
2496
2497	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2498	/** Automatically tracked physical memory mapping set.
2499	* Ring-0 and strict raw-mode builds. */
2500	PGMMAPSET AutoSet;
2501	#endif
2502
2503	/** A20 gate mask.
2504	* Our current approach to A20 emulation is to let REM do it and don't bother
2505	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2506	* But whould need arrise, we'll subject physical addresses to this mask. */
2507	RTGCPHYS GCPhysA20Mask;
2508	/** A20 gate state - boolean! */
2509	bool fA20Enabled;
2510
2511	/** What needs syncing (PGM_SYNC_*).
2512	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2513	* PGMFlushTLB, and PGMR3Load. */
2514	RTUINT fSyncFlags;
2515
2516	/** The shadow paging mode. */
2517	PGMMODE enmShadowMode;
2518	/** The guest paging mode. */
2519	PGMMODE enmGuestMode;
2520
2521	/** The current physical address representing in the guest CR3 register. */
2522	RTGCPHYS GCPhysCR3;
2523
2524	/** @name 32-bit Guest Paging.
2525	* @{ */
2526	/** The guest's page directory, R3 pointer. */
2527	R3PTRTYPE(PX86PD) pGst32BitPdR3;
2528	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2529	/** The guest's page directory, R0 pointer. */
2530	R0PTRTYPE(PX86PD) pGst32BitPdR0;
2531	#endif
2532	/** The guest's page directory, static RC mapping. */
2533	RCPTRTYPE(PX86PD) pGst32BitPdRC;
2534	/** @} */
2535
2536	/** @name PAE Guest Paging.
2537	* @{ */
2538	/** The guest's page directory pointer table, static RC mapping. */
2539	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
2540	/** The guest's page directory pointer table, R3 pointer. */
2541	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
2542	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2543	/** The guest's page directory pointer table, R0 pointer. */
2544	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
2545	#endif
2546
2547	/** The guest's page directories, R3 pointers.
2548	* These are individual pointers and don't have to be adjecent.
2549	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2550	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
2551	/** The guest's page directories, R0 pointers.
2552	* Same restrictions as apGstPaePDsR3. */
2553	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2554	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
2555	#endif
2556	/** The guest's page directories, static GC mapping.
2557	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
2558	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2559	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
2560	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
2561	RTGCPHYS aGCPhysGstPaePDs[4];
2562	/** The physical addresses of the monitored guest page directories (PAE). */
2563	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
2564	/** @} */
2565
2566	/** @name AMD64 Guest Paging.
2567	* @{ */
2568	/** The guest's page directory pointer table, R3 pointer. */
2569	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
2570	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2571	/** The guest's page directory pointer table, R0 pointer. */
2572	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
2573	#endif
2574	/** @} */
2575
2576	/** Pointer to the page of the current active CR3 - R3 Ptr. */
2577	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
2578	/** Pointer to the page of the current active CR3 - R0 Ptr. */
2579	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
2580	/** Pointer to the page of the current active CR3 - RC Ptr. */
2581	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
2582	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
2583	uint32_t iShwUser;
2584	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
2585	uint32_t iShwUserTable;
2586	# if HC_ARCH_BITS == 64
2587	RTRCPTR alignment6; /*< structure size alignment. /
2588	# endif
2589	/** @} */
2590
2591	/** @name Function pointers for Shadow paging.
2592	* @{
2593	*/
2594	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2595	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2596	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2597	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2598
2599	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2600	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2601
2602	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2603	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2604
2605	/** @} */
2606
2607	/** @name Function pointers for Guest paging.
2608	* @{
2609	*/
2610	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2611	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2612	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2613	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2614	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2615	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2616	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2617	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2618	#if HC_ARCH_BITS == 64
2619	RTRCPTR alignment3; /*< structure size alignment. /
2620	#endif
2621
2622	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2623	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2624	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2625	/** @} */
2626
2627	/** @name Function pointers for Both Shadow and Guest paging.
2628	* @{
2629	*/
2630	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2631	/* no pfnR3BthTrap0eHandler */
2632	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2633	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2634	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2635	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2636	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2637	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2638	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2639	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2640
2641	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2642	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2643	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2644	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2645	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2646	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2647	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2648	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2649	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2650
2651	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2652	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2653	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2654	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2655	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2656	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2657	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2658	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2659	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2660	#if HC_ARCH_BITS == 64
2661	RTRCPTR alignment2; /*< structure size alignment. /
2662	#endif
2663	/** @} */
2664
2665	/** @name Release Statistics
2666	* @{ */
2667	/** The number of times the guest has switched mode since last reset or statistics reset. */
2668	STAMCOUNTER cGuestModeChanges;
2669	/** @} */
2670
2671	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2672	/** @name Statistics
2673	* @{ */
2674	/** RC: Which statistic this \#PF should be attributed to. */
2675	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
2676	RTRCPTR padding0;
2677	/** R0: Which statistic this \#PF should be attributed to. */
2678	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
2679	RTR0PTR padding1;
2680
2681	/* Common */
2682	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
2683	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
2684
2685	/* R0 only: */
2686	STAMCOUNTER StatR0DynMapMigrateInvlPg; /*< R0: invlpg in PGMDynMapMigrateAutoSet. /
2687	STAMPROFILE StatR0DynMapGCPageInl; /*< R0: Calls to pgmR0DynMapGCPageInlined. /
2688	STAMCOUNTER StatR0DynMapGCPageInlHits; /*< R0: Hash table lookup hits. /
2689	STAMCOUNTER StatR0DynMapGCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2690	STAMCOUNTER StatR0DynMapGCPageInlRamHits; /*< R0: 1st ram range hits. /
2691	STAMCOUNTER StatR0DynMapGCPageInlRamMisses; /*< R0: 1st ram range misses, takes slow path. /
2692	STAMPROFILE StatR0DynMapHCPageInl; /*< R0: Calls to pgmR0DynMapHCPageInlined. /
2693	STAMCOUNTER StatR0DynMapHCPageInlHits; /*< R0: Hash table lookup hits. /
2694	STAMCOUNTER StatR0DynMapHCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2695	STAMPROFILE StatR0DynMapHCPage; /*< R0: Calls to PGMDynMapHCPage. /
2696	STAMCOUNTER StatR0DynMapSetOptimize; /*< R0: Calls to pgmDynMapOptimizeAutoSet. /
2697	STAMCOUNTER StatR0DynMapSetSearchFlushes; /*< R0: Set search restorting to subset flushes. /
2698	STAMCOUNTER StatR0DynMapSetSearchHits; /*< R0: Set search hits. /
2699	STAMCOUNTER StatR0DynMapSetSearchMisses; /*< R0: Set search misses. /
2700	STAMCOUNTER StatR0DynMapPage; /*< R0: Calls to pgmR0DynMapPage. /
2701	STAMCOUNTER StatR0DynMapPageHits0; /*< R0: Hits at iPage+0. /
2702	STAMCOUNTER StatR0DynMapPageHits1; /*< R0: Hits at iPage+1. /
2703	STAMCOUNTER StatR0DynMapPageHits2; /*< R0: Hits at iPage+2. /
2704	STAMCOUNTER StatR0DynMapPageInvlPg; /*< R0: invlpg. /
2705	STAMCOUNTER StatR0DynMapPageSlow; /*< R0: Calls to pgmR0DynMapPageSlow. /
2706	STAMCOUNTER StatR0DynMapPageSlowLoopHits; /*< R0: Hits in the pgmR0DynMapPageSlow search loop. /
2707	STAMCOUNTER StatR0DynMapPageSlowLoopMisses; /*< R0: Misses in the pgmR0DynMapPageSlow search loop. /
2708	//STAMCOUNTER StatR0DynMapPageSlowLostHits; /*< R0: Lost hits. /
2709	STAMCOUNTER StatR0DynMapSubsets; /*< R0: Times PGMDynMapPushAutoSubset was called. /
2710	STAMCOUNTER StatR0DynMapPopFlushes; /*< R0: Times PGMDynMapPopAutoSubset flushes the subset. /
2711	STAMCOUNTER aStatR0DynMapSetSize[11]; /*< R0: Set size distribution. /
2712
2713	/* RZ only: */
2714	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
2715	STAMPROFILE StatRZTrap0eTimeCheckPageFault;
2716	STAMPROFILE StatRZTrap0eTimeSyncPT;
2717	STAMPROFILE StatRZTrap0eTimeMapping;
2718	STAMPROFILE StatRZTrap0eTimeOutOfSync;
2719	STAMPROFILE StatRZTrap0eTimeHandlers;
2720	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
2721	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
2722	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
2723	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
2724	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
2725	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
2726	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
2727	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
2728	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
2729	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
2730	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
2731	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
2732	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
2733	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
2734	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
2735	STAMCOUNTER StatRZTrap0eHandlersPhysical; /*< RC/R0: Number of traps due to physical access handlers. /
2736	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
2737	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
2738	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
2739	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
2740	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
2741	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: #PF err kind /
2742	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: #PF err kind /
2743	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: #PF err kind /
2744	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: #PF err kind /
2745	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: #PF err kind /
2746	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: #PF err kind /
2747	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: #PF err kind /
2748	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: #PF err kind /
2749	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: #PF err kind /
2750	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: #PF err kind /
2751	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: #PF err kind /
2752	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest #PFs. /
2753	STAMCOUNTER StatRZTrap0eGuestPFUnh; /*< RC/R0: Real guest #PF ending up at the end of the #PF code. /
2754	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest #PF to HMA or other mapping. /
2755	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
2756	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
2757	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the #PFs. /
2758	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
2759	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
2760	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
2761	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
2762	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
2763
2764	/* HC - R3 and (maybe) R0: */
2765
2766	/* RZ & R3: */
2767	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
2768	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
2769	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
2770	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
2771	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
2772	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
2773	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
2774	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
2775	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
2776	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
2777	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
2778	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
2779	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
2780	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
2781	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2782	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
2783	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
2784	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
2785	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2786	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2787	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
2788	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
2789	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
2790	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
2791	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
2792	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
2793	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
2794	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
2795	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
2796	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2797	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2798	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
2799	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2800	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2801	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2802	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2803	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
2804	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
2805	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2806	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2807	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2808	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2809	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
2810
2811	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
2812	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
2813	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
2814	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
2815	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
2816	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
2817	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
2818	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
2819	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
2820	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
2821	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
2822	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
2823	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
2824	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
2825	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2826	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
2827	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
2828	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
2829	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2830	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2831	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
2832	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
2833	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
2834	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
2835	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
2836	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
2837	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
2838	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
2839	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
2840	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2841	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
2842	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2843	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2844	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2845	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2846	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2847	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
2848	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
2849	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2850	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2851	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2852	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2853	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
2854	/** @} */
2855	#endif /* VBOX_WITH_STATISTICS */
2856	} PGMCPU;
2857	/** Pointer to the per-cpu PGM data. */
2858	typedef PGMCPU *PPGMCPU;
2859
2860
2861	/** @name PGM::fSyncFlags Flags
2862	* @{
2863	*/
2864	/** Updates the virtual access handler state bit in PGMPAGE. */
2865	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
2866	/** Always sync CR3. */
2867	#define PGM_SYNC_ALWAYS RT_BIT(1)
2868	/** Check monitoring on next CR3 (re)load and invalidate page.
2869	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
2870	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
2871	/** Check guest mapping in SyncCR3. */
2872	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
2873	/** Clear the page pool (a light weight flush). */
2874	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL RT_BIT(8)
2875	/** @} */
2876
2877
2878	__BEGIN_DECLS
2879
2880	int pgmLock(PVM pVM);
2881	void pgmUnlock(PVM pVM);
2882
2883	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
2884	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
2885	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
2886	void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping);
2887	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2888
2889	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
2890	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
2891	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage);
2892	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
2893	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
2894	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
2895	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
2896	#else
2897	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
2898	#endif
2899	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2900
2901
2902	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2903	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
2904	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2905	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2906	int pgmPhysPageMakeWritableUnlocked(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2907	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv);
2908	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
2909	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
2910	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
2911	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
2912	#ifdef IN_RING3
2913	void pgmR3PhysRelinkRamRanges(PVM pVM);
2914	int pgmR3PhysRamPreAllocate(PVM pVM);
2915	int pgmR3PhysRamReset(PVM pVM);
2916	int pgmR3PhysRomReset(PVM pVM);
2917	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
2918
2919	int pgmR3PoolInit(PVM pVM);
2920	void pgmR3PoolRelocate(PVM pVM);
2921	void pgmR3PoolReset(PVM pVM);
2922
2923	#endif /* IN_RING3 */
2924	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2925	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
2926	#endif
2927	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage);
2928	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
2929	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
2930	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2931	void pgmPoolFlushAll(PVM pVM);
2932	void pgmPoolClearAll(PVM pVM);
2933	int pgmPoolSyncCR3(PVM pVM);
2934	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs);
2935	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT);
2936	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage);
2937	#ifdef PGMPOOL_WITH_MONITORING
2938	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu);
2939	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2940	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2941	#endif
2942
2943	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
2944	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
2945
2946	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
2947	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
2948	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2949	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2950
2951	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2952	#ifndef IN_RC
2953	int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2954	#endif
2955	int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
2956
2957	PX86PD pgmGstLazyMap32BitPD(PPGMCPU pPGM);
2958	PX86PDPT pgmGstLazyMapPaePDPT(PPGMCPU pPGM);
2959	PX86PDPAE pgmGstLazyMapPaePD(PPGMCPU pPGM, uint32_t iPdpt);
2960	PX86PML4 pgmGstLazyMapPml4(PPGMCPU pPGM);
2961
2962	/* Forward decl */
2963	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
2964
2965	__END_DECLS
2966
2967
2968	/**
2969	* Gets the PGMRAMRANGE structure for a guest page.
2970	*
2971	* @returns Pointer to the RAM range on success.
2972	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
2973	*
2974	* @param pPGM PGM handle.
2975	* @param GCPhys The GC physical address.
2976	*/
2977	DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
2978	{
2979	/*
2980	* Optimize for the first range.
2981	*/
2982	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
2983	RTGCPHYS off = GCPhys - pRam->GCPhys;
2984	if (RT_UNLIKELY(off >= pRam->cb))
2985	{
2986	do
2987	{
2988	pRam = pRam->CTX_SUFF(pNext);
2989	if (RT_UNLIKELY(!pRam))
2990	break;
2991	off = GCPhys - pRam->GCPhys;
2992	} while (off >= pRam->cb);
2993	}
2994	return pRam;
2995	}
2996
2997
2998	/**
2999	* Gets the PGMPAGE structure for a guest page.
3000	*
3001	* @returns Pointer to the page on success.
3002	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3003	*
3004	* @param pPGM PGM handle.
3005	* @param GCPhys The GC physical address.
3006	*/
3007	DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
3008	{
3009	/*
3010	* Optimize for the first range.
3011	*/
3012	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3013	RTGCPHYS off = GCPhys - pRam->GCPhys;
3014	if (RT_UNLIKELY(off >= pRam->cb))
3015	{
3016	do
3017	{
3018	pRam = pRam->CTX_SUFF(pNext);
3019	if (RT_UNLIKELY(!pRam))
3020	return NULL;
3021	off = GCPhys - pRam->GCPhys;
3022	} while (off >= pRam->cb);
3023	}
3024	return &pRam->aPages[off >> PAGE_SHIFT];
3025	}
3026
3027
3028	/**
3029	* Gets the PGMPAGE structure for a guest page.
3030	*
3031	* Old Phys code: Will make sure the page is present.
3032	*
3033	* @returns VBox status code.
3034	* @retval VINF_SUCCESS and a valid *ppPage on success.
3035	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3036	*
3037	* @param pPGM PGM handle.
3038	* @param GCPhys The GC physical address.
3039	* @param ppPage Where to store the page poitner on success.
3040	*/
3041	DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
3042	{
3043	/*
3044	* Optimize for the first range.
3045	*/
3046	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3047	RTGCPHYS off = GCPhys - pRam->GCPhys;
3048	if (RT_UNLIKELY(off >= pRam->cb))
3049	{
3050	do
3051	{
3052	pRam = pRam->CTX_SUFF(pNext);
3053	if (RT_UNLIKELY(!pRam))
3054	{
3055	ppPage = NULL; / avoid incorrect and very annoying GCC warnings */
3056	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3057	}
3058	off = GCPhys - pRam->GCPhys;
3059	} while (off >= pRam->cb);
3060	}
3061	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3062	return VINF_SUCCESS;
3063	}
3064
3065
3066
3067
3068	/**
3069	* Gets the PGMPAGE structure for a guest page.
3070	*
3071	* Old Phys code: Will make sure the page is present.
3072	*
3073	* @returns VBox status code.
3074	* @retval VINF_SUCCESS and a valid *ppPage on success.
3075	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3076	*
3077	* @param pPGM PGM handle.
3078	* @param GCPhys The GC physical address.
3079	* @param ppPage Where to store the page poitner on success.
3080	* @param ppRamHint Where to read and store the ram list hint.
3081	* The caller initializes this to NULL before the call.
3082	*/
3083	DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
3084	{
3085	RTGCPHYS off;
3086	PPGMRAMRANGE pRam = *ppRamHint;
3087	if ( !pRam
3088	\|\| RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
3089	{
3090	pRam = pPGM->CTX_SUFF(pRamRanges);
3091	off = GCPhys - pRam->GCPhys;
3092	if (RT_UNLIKELY(off >= pRam->cb))
3093	{
3094	do
3095	{
3096	pRam = pRam->CTX_SUFF(pNext);
3097	if (RT_UNLIKELY(!pRam))
3098	{
3099	ppPage = NULL; / Kill the incorrect and extremely annoying GCC warnings. */
3100	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3101	}
3102	off = GCPhys - pRam->GCPhys;
3103	} while (off >= pRam->cb);
3104	}
3105	*ppRamHint = pRam;
3106	}
3107	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3108	return VINF_SUCCESS;
3109	}
3110
3111
3112	/**
3113	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3114	*
3115	* @returns Pointer to the page on success.
3116	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3117	*
3118	* @param pPGM PGM handle.
3119	* @param GCPhys The GC physical address.
3120	* @param ppRam Where to store the pointer to the PGMRAMRANGE.
3121	*/
3122	DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
3123	{
3124	/*
3125	* Optimize for the first range.
3126	*/
3127	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3128	RTGCPHYS off = GCPhys - pRam->GCPhys;
3129	if (RT_UNLIKELY(off >= pRam->cb))
3130	{
3131	do
3132	{
3133	pRam = pRam->CTX_SUFF(pNext);
3134	if (RT_UNLIKELY(!pRam))
3135	return NULL;
3136	off = GCPhys - pRam->GCPhys;
3137	} while (off >= pRam->cb);
3138	}
3139	*ppRam = pRam;
3140	return &pRam->aPages[off >> PAGE_SHIFT];
3141	}
3142
3143
3144	/**
3145	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3146	*
3147	* @returns Pointer to the page on success.
3148	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3149	*
3150	* @param pPGM PGM handle.
3151	* @param GCPhys The GC physical address.
3152	* @param ppPage Where to store the pointer to the PGMPAGE structure.
3153	* @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
3154	*/
3155	DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
3156	{
3157	/*
3158	* Optimize for the first range.
3159	*/
3160	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3161	RTGCPHYS off = GCPhys - pRam->GCPhys;
3162	if (RT_UNLIKELY(off >= pRam->cb))
3163	{
3164	do
3165	{
3166	pRam = pRam->CTX_SUFF(pNext);
3167	if (RT_UNLIKELY(!pRam))
3168	{
3169	ppRam = NULL; / Shut up silly GCC warnings. */
3170	ppPage = NULL; / ditto */
3171	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3172	}
3173	off = GCPhys - pRam->GCPhys;
3174	} while (off >= pRam->cb);
3175	}
3176	*ppRam = pRam;
3177	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3178	return VINF_SUCCESS;
3179	}
3180
3181
3182	/**
3183	* Convert GC Phys to HC Phys.
3184	*
3185	* @returns VBox status.
3186	* @param pPGM PGM handle.
3187	* @param GCPhys The GC physical address.
3188	* @param pHCPhys Where to store the corresponding HC physical address.
3189	*
3190	* @deprecated Doesn't deal with zero, shared or write monitored pages.
3191	* Avoid when writing new code!
3192	*/
3193	DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
3194	{
3195	PPGMPAGE pPage;
3196	int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
3197	if (RT_FAILURE(rc))
3198	return rc;
3199	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
3200	return VINF_SUCCESS;
3201	}
3202
3203	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3204
3205	/**
3206	* Inlined version of the ring-0 version of PGMDynMapHCPage that
3207	* optimizes access to pages already in the set.
3208	*
3209	* @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
3210	* @param pPGM Pointer to the PVM instance data.
3211	* @param HCPhys The physical address of the page.
3212	* @param ppv Where to store the mapping address.
3213	*/
3214	DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
3215	{
3216	PVM pVM = PGM2VM(pPGM);
3217	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3218	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3219
3220	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
3221	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3222	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3223
3224	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3225	unsigned iEntry = pSet->aiHashTable[iHash];
3226	if ( iEntry < pSet->cEntries
3227	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3228	{
3229	*ppv = pSet->aEntries[iEntry].pvPage;
3230	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
3231	}
3232	else
3233	{
3234	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
3235	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3236	}
3237
3238	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
3239	return VINF_SUCCESS;
3240	}
3241
3242
3243	/**
3244	* Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
3245	* access to pages already in the set.
3246	*
3247	* @returns See PGMDynMapGCPage.
3248	* @param pPGM Pointer to the PVM instance data.
3249	* @param HCPhys The physical address of the page.
3250	* @param ppv Where to store the mapping address.
3251	*/
3252	DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3253	{
3254	PVM pVM = PGM2VM(pPGM);
3255	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3256
3257	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3258	Assert(!(GCPhys & PAGE_OFFSET_MASK));
3259
3260	/*
3261	* Get the ram range.
3262	*/
3263	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3264	RTGCPHYS off = GCPhys - pRam->GCPhys;
3265	if (RT_UNLIKELY(off >= pRam->cb
3266	/** @todo \|\| page state stuff */))
3267	{
3268	/* This case is not counted into StatR0DynMapGCPageInl. */
3269	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3270	return PGMDynMapGCPage(pVM, GCPhys, ppv);
3271	}
3272
3273	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3274	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3275
3276	/*
3277	* pgmR0DynMapHCPageInlined with out stats.
3278	*/
3279	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3280	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3281	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3282
3283	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3284	unsigned iEntry = pSet->aiHashTable[iHash];
3285	if ( iEntry < pSet->cEntries
3286	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3287	{
3288	*ppv = pSet->aEntries[iEntry].pvPage;
3289	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3290	}
3291	else
3292	{
3293	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3294	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3295	}
3296
3297	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3298	return VINF_SUCCESS;
3299	}
3300
3301	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3302	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3303
3304	/**
3305	* Maps the page into current context (RC and maybe R0).
3306	*
3307	* @returns pointer to the mapping.
3308	* @param pVM Pointer to the PGM instance data.
3309	* @param pPage The page.
3310	*/
3311	DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
3312	{
3313	if (pPage->idx >= PGMPOOL_IDX_FIRST)
3314	{
3315	Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
3316	void *pv;
3317	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3318	pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
3319	# else
3320	PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
3321	# endif
3322	return pv;
3323	}
3324	AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
3325	}
3326
3327	/**
3328	* Temporarily maps one host page specified by HC physical address, returning
3329	* pointer within the page.
3330	*
3331	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
3332	* reused after 8 mappings (or perhaps a few more if you score with the cache).
3333	*
3334	* @returns The address corresponding to HCPhys.
3335	* @param pPGM Pointer to the PVM instance data.
3336	* @param HCPhys HC Physical address of the page.
3337	*/
3338	DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
3339	{
3340	void *pv;
3341	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3342	pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3343	# else
3344	PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3345	# endif
3346	pv = (void *)((uintptr_t)pv \| (HCPhys & PAGE_OFFSET_MASK));
3347	return pv;
3348	}
3349
3350	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 \|\| IN_RC */
3351	#ifndef IN_RC
3352
3353	/**
3354	* Queries the Physical TLB entry for a physical guest page,
3355	* attempting to load the TLB entry if necessary.
3356	*
3357	* @returns VBox status code.
3358	* @retval VINF_SUCCESS on success
3359	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3360	*
3361	* @param pPGM The PGM instance handle.
3362	* @param GCPhys The address of the guest page.
3363	* @param ppTlbe Where to store the pointer to the TLB entry.
3364	*/
3365	DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3366	{
3367	int rc;
3368	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3369	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3370	{
3371	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3372	rc = VINF_SUCCESS;
3373	}
3374	else
3375	rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
3376	*ppTlbe = pTlbe;
3377	return rc;
3378	}
3379
3380
3381	/**
3382	* Queries the Physical TLB entry for a physical guest page,
3383	* attempting to load the TLB entry if necessary.
3384	*
3385	* @returns VBox status code.
3386	* @retval VINF_SUCCESS on success
3387	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3388	*
3389	* @param pPGM The PGM instance handle.
3390	* @param pPage Pointer to the PGMPAGE structure corresponding to
3391	* GCPhys.
3392	* @param GCPhys The address of the guest page.
3393	* @param ppTlbe Where to store the pointer to the TLB entry.
3394	*/
3395	DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3396	{
3397	int rc;
3398	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3399	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3400	{
3401	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3402	rc = VINF_SUCCESS;
3403	}
3404	else
3405	rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
3406	*ppTlbe = pTlbe;
3407	return rc;
3408	}
3409
3410	#endif /* !IN_RC */
3411
3412	/**
3413	* Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
3414	* Takes PSE-36 into account.
3415	*
3416	* @returns guest physical address
3417	* @param pPGM Pointer to the PGM instance data.
3418	* @param Pde Guest Pde
3419	*/
3420	DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
3421	{
3422	RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
3423	GCPhys \|= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
3424
3425	return GCPhys & pPGM->GCPhys4MBPSEMask;
3426	}
3427
3428
3429	/**
3430	* Gets the page directory entry for the specified address (32-bit paging).
3431	*
3432	* @returns The page directory entry in question.
3433	* @param pPGM Pointer to the PGM instance data.
3434	* @param GCPtr The address.
3435	*/
3436	DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3437	{
3438	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3439	PCX86PD pGuestPD = NULL;
3440	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3441	if (RT_FAILURE(rc))
3442	{
3443	X86PDE ZeroPde = {0};
3444	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
3445	}
3446	#else
3447	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3448	# ifdef IN_RING3
3449	if (!pGuestPD)
3450	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3451	# endif
3452	#endif
3453	return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3454	}
3455
3456
3457	/**
3458	* Gets the address of a specific page directory entry (32-bit paging).
3459	*
3460	* @returns Pointer the page directory entry in question.
3461	* @param pPGM Pointer to the PGM instance data.
3462	* @param GCPtr The address.
3463	*/
3464	DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3465	{
3466	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3467	PX86PD pGuestPD = NULL;
3468	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3469	AssertRCReturn(rc, NULL);
3470	#else
3471	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3472	# ifdef IN_RING3
3473	if (!pGuestPD)
3474	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3475	# endif
3476	#endif
3477	return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3478	}
3479
3480
3481	/**
3482	* Gets the address the guest page directory (32-bit paging).
3483	*
3484	* @returns Pointer the page directory entry in question.
3485	* @param pPGM Pointer to the PGM instance data.
3486	*/
3487	DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
3488	{
3489	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3490	PX86PD pGuestPD = NULL;
3491	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3492	AssertRCReturn(rc, NULL);
3493	#else
3494	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3495	# ifdef IN_RING3
3496	if (!pGuestPD)
3497	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3498	# endif
3499	#endif
3500	return pGuestPD;
3501	}
3502
3503
3504	/**
3505	* Gets the guest page directory pointer table.
3506	*
3507	* @returns Pointer to the page directory in question.
3508	* @returns NULL if the page directory is not present or on an invalid page.
3509	* @param pPGM Pointer to the PGM instance data.
3510	*/
3511	DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
3512	{
3513	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3514	PX86PDPT pGuestPDPT = NULL;
3515	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3516	AssertRCReturn(rc, NULL);
3517	#else
3518	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3519	# ifdef IN_RING3
3520	if (!pGuestPDPT)
3521	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3522	# endif
3523	#endif
3524	return pGuestPDPT;
3525	}
3526
3527
3528	/**
3529	* Gets the guest page directory pointer table entry for the specified address.
3530	*
3531	* @returns Pointer to the page directory in question.
3532	* @returns NULL if the page directory is not present or on an invalid page.
3533	* @param pPGM Pointer to the PGM instance data.
3534	* @param GCPtr The address.
3535	*/
3536	DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3537	{
3538	AssertGCPtr32(GCPtr);
3539
3540	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3541	PX86PDPT pGuestPDPT = 0;
3542	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3543	AssertRCReturn(rc, 0);
3544	#else
3545	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3546	# ifdef IN_RING3
3547	if (!pGuestPDPT)
3548	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3549	# endif
3550	#endif
3551	return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
3552	}
3553
3554
3555	/**
3556	* Gets the page directory for the specified address.
3557	*
3558	* @returns Pointer to the page directory in question.
3559	* @returns NULL if the page directory is not present or on an invalid page.
3560	* @param pPGM Pointer to the PGM instance data.
3561	* @param GCPtr The address.
3562	*/
3563	DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
3564	{
3565	AssertGCPtr32(GCPtr);
3566
3567	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3568	AssertReturn(pGuestPDPT, NULL);
3569	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3570	if (pGuestPDPT->a[iPdpt].n.u1Present)
3571	{
3572	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3573	PX86PDPAE pGuestPD = NULL;
3574	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3575	AssertRCReturn(rc, NULL);
3576	#else
3577	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3578	if ( !pGuestPD
3579	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3580	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3581	#endif
3582	return pGuestPD;
3583	/* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
3584	}
3585	return NULL;
3586	}
3587
3588
3589	/**
3590	* Gets the page directory entry for the specified address.
3591	*
3592	* @returns Pointer to the page directory entry in question.
3593	* @returns NULL if the page directory is not present or on an invalid page.
3594	* @param pPGM Pointer to the PGM instance data.
3595	* @param GCPtr The address.
3596	*/
3597	DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3598	{
3599	AssertGCPtr32(GCPtr);
3600
3601	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3602	AssertReturn(pGuestPDPT, NULL);
3603	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3604	if (pGuestPDPT->a[iPdpt].n.u1Present)
3605	{
3606	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3607	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3608	PX86PDPAE pGuestPD = NULL;
3609	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3610	AssertRCReturn(rc, NULL);
3611	#else
3612	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3613	if ( !pGuestPD
3614	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3615	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3616	#endif
3617	return &pGuestPD->a[iPD];
3618	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
3619	}
3620	return NULL;
3621	}
3622
3623
3624	/**
3625	* Gets the page directory entry for the specified address.
3626	*
3627	* @returns The page directory entry in question.
3628	* @returns A non-present entry if the page directory is not present or on an invalid page.
3629	* @param pPGM Pointer to the PGM instance data.
3630	* @param GCPtr The address.
3631	*/
3632	DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3633	{
3634	AssertGCPtr32(GCPtr);
3635	X86PDEPAE ZeroPde = {0};
3636	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3637	if (RT_LIKELY(pGuestPDPT))
3638	{
3639	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3640	if (pGuestPDPT->a[iPdpt].n.u1Present)
3641	{
3642	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3643	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3644	PX86PDPAE pGuestPD = NULL;
3645	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3646	AssertRCReturn(rc, ZeroPde);
3647	#else
3648	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3649	if ( !pGuestPD
3650	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3651	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3652	#endif
3653	return pGuestPD->a[iPD];
3654	}
3655	}
3656	return ZeroPde;
3657	}
3658
3659
3660	/**
3661	* Gets the page directory pointer table entry for the specified address
3662	* and returns the index into the page directory
3663	*
3664	* @returns Pointer to the page directory in question.
3665	* @returns NULL if the page directory is not present or on an invalid page.
3666	* @param pPGM Pointer to the PGM instance data.
3667	* @param GCPtr The address.
3668	* @param piPD Receives the index into the returned page directory
3669	* @param pPdpe Receives the page directory pointer entry. Optional.
3670	*/
3671	DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
3672	{
3673	AssertGCPtr32(GCPtr);
3674
3675	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3676	AssertReturn(pGuestPDPT, NULL);
3677	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3678	if (pPdpe)
3679	*pPdpe = pGuestPDPT->a[iPdpt];
3680	if (pGuestPDPT->a[iPdpt].n.u1Present)
3681	{
3682	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3683	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3684	PX86PDPAE pGuestPD = NULL;
3685	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3686	AssertRCReturn(rc, NULL);
3687	#else
3688	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3689	if ( !pGuestPD
3690	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3691	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3692	#endif
3693	*piPD = iPD;
3694	return pGuestPD;
3695	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
3696	}
3697	return NULL;
3698	}
3699
3700	#ifndef IN_RC
3701
3702	/**
3703	* Gets the page map level-4 pointer for the guest.
3704	*
3705	* @returns Pointer to the PML4 page.
3706	* @param pPGM Pointer to the PGM instance data.
3707	*/
3708	DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
3709	{
3710	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3711	PX86PML4 pGuestPml4;
3712	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3713	AssertRCReturn(rc, NULL);
3714	#else
3715	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3716	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3717	if (!pGuestPml4)
3718	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3719	# endif
3720	Assert(pGuestPml4);
3721	#endif
3722	return pGuestPml4;
3723	}
3724
3725
3726	/**
3727	* Gets the pointer to a page map level-4 entry.
3728	*
3729	* @returns Pointer to the PML4 entry.
3730	* @param pPGM Pointer to the PGM instance data.
3731	* @param iPml4 The index.
3732	*/
3733	DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
3734	{
3735	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3736	PX86PML4 pGuestPml4;
3737	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3738	AssertRCReturn(rc, NULL);
3739	#else
3740	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3741	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3742	if (!pGuestPml4)
3743	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3744	# endif
3745	Assert(pGuestPml4);
3746	#endif
3747	return &pGuestPml4->a[iPml4];
3748	}
3749
3750
3751	/**
3752	* Gets a page map level-4 entry.
3753	*
3754	* @returns The PML4 entry.
3755	* @param pPGM Pointer to the PGM instance data.
3756	* @param iPml4 The index.
3757	*/
3758	DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
3759	{
3760	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3761	PX86PML4 pGuestPml4;
3762	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3763	if (RT_FAILURE(rc))
3764	{
3765	X86PML4E ZeroPml4e = {0};
3766	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
3767	}
3768	#else
3769	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3770	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3771	if (!pGuestPml4)
3772	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3773	# endif
3774	Assert(pGuestPml4);
3775	#endif
3776	return pGuestPml4->a[iPml4];
3777	}
3778
3779
3780	/**
3781	* Gets the page directory pointer entry for the specified address.
3782	*
3783	* @returns Pointer to the page directory pointer entry in question.
3784	* @returns NULL if the page directory is not present or on an invalid page.
3785	* @param pPGM Pointer to the PGM instance data.
3786	* @param GCPtr The address.
3787	* @param ppPml4e Page Map Level-4 Entry (out)
3788	*/
3789	DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
3790	{
3791	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3792	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3793	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3794	if (pPml4e->n.u1Present)
3795	{
3796	PX86PDPT pPdpt;
3797	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
3798	AssertRCReturn(rc, NULL);
3799
3800	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3801	return &pPdpt->a[iPdpt];
3802	}
3803	return NULL;
3804	}
3805
3806
3807	/**
3808	* Gets the page directory entry for the specified address.
3809	*
3810	* @returns The page directory entry in question.
3811	* @returns A non-present entry if the page directory is not present or on an invalid page.
3812	* @param pPGM Pointer to the PGM instance data.
3813	* @param GCPtr The address.
3814	* @param ppPml4e Page Map Level-4 Entry (out)
3815	* @param pPdpe Page directory pointer table entry (out)
3816	*/
3817	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
3818	{
3819	X86PDEPAE ZeroPde = {0};
3820	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3821	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3822	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3823	if (pPml4e->n.u1Present)
3824	{
3825	PCX86PDPT pPdptTemp;
3826	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3827	AssertRCReturn(rc, ZeroPde);
3828
3829	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3830	*pPdpe = pPdptTemp->a[iPdpt];
3831	if (pPdptTemp->a[iPdpt].n.u1Present)
3832	{
3833	PCX86PDPAE pPD;
3834	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3835	AssertRCReturn(rc, ZeroPde);
3836
3837	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3838	return pPD->a[iPD];
3839	}
3840	}
3841
3842	return ZeroPde;
3843	}
3844
3845
3846	/**
3847	* Gets the page directory entry for the specified address.
3848	*
3849	* @returns The page directory entry in question.
3850	* @returns A non-present entry if the page directory is not present or on an invalid page.
3851	* @param pPGM Pointer to the PGM instance data.
3852	* @param GCPtr The address.
3853	*/
3854	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3855	{
3856	X86PDEPAE ZeroPde = {0};
3857	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3858	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3859	if (pGuestPml4->a[iPml4].n.u1Present)
3860	{
3861	PCX86PDPT pPdptTemp;
3862	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3863	AssertRCReturn(rc, ZeroPde);
3864
3865	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3866	if (pPdptTemp->a[iPdpt].n.u1Present)
3867	{
3868	PCX86PDPAE pPD;
3869	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3870	AssertRCReturn(rc, ZeroPde);
3871
3872	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3873	return pPD->a[iPD];
3874	}
3875	}
3876	return ZeroPde;
3877	}
3878
3879
3880	/**
3881	* Gets the page directory entry for the specified address.
3882	*
3883	* @returns Pointer to the page directory entry in question.
3884	* @returns NULL if the page directory is not present or on an invalid page.
3885	* @param pPGM Pointer to the PGM instance data.
3886	* @param GCPtr The address.
3887	*/
3888	DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3889	{
3890	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3891	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3892	if (pGuestPml4->a[iPml4].n.u1Present)
3893	{
3894	PCX86PDPT pPdptTemp;
3895	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3896	AssertRCReturn(rc, NULL);
3897
3898	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3899	if (pPdptTemp->a[iPdpt].n.u1Present)
3900	{
3901	PX86PDPAE pPD;
3902	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3903	AssertRCReturn(rc, NULL);
3904
3905	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3906	return &pPD->a[iPD];
3907	}
3908	}
3909	return NULL;
3910	}
3911
3912
3913	/**
3914	* Gets the GUEST page directory pointer for the specified address.
3915	*
3916	* @returns The page directory in question.
3917	* @returns NULL if the page directory is not present or on an invalid page.
3918	* @param pPGM Pointer to the PGM instance data.
3919	* @param GCPtr The address.
3920	* @param ppPml4e Page Map Level-4 Entry (out)
3921	* @param pPdpe Page directory pointer table entry (out)
3922	* @param piPD Receives the index into the returned page directory
3923	*/
3924	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPE pPdpe, unsigned piPD)
3925	{
3926	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3927	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3928	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3929	if (pPml4e->n.u1Present)
3930	{
3931	PCX86PDPT pPdptTemp;
3932	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3933	AssertRCReturn(rc, NULL);
3934
3935	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3936	*pPdpe = pPdptTemp->a[iPdpt];
3937	if (pPdptTemp->a[iPdpt].n.u1Present)
3938	{
3939	PX86PDPAE pPD;
3940	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3941	AssertRCReturn(rc, NULL);
3942
3943	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3944	return pPD;
3945	}
3946	}
3947	return 0;
3948	}
3949
3950	#endif /* !IN_RC */
3951
3952	/**
3953	* Gets the shadow page directory, 32-bit.
3954	*
3955	* @returns Pointer to the shadow 32-bit PD.
3956	* @param pPGM Pointer to the PGM instance data.
3957	*/
3958	DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
3959	{
3960	return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
3961	}
3962
3963
3964	/**
3965	* Gets the shadow page directory entry for the specified address, 32-bit.
3966	*
3967	* @returns Shadow 32-bit PDE.
3968	* @param pPGM Pointer to the PGM instance data.
3969	* @param GCPtr The address.
3970	*/
3971	DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3972	{
3973	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
3974
3975	PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
3976	if (!pShwPde)
3977	{
3978	X86PDE ZeroPde = {0};
3979	return ZeroPde;
3980	}
3981	return pShwPde->a[iPd];
3982	}
3983
3984
3985	/**
3986	* Gets the pointer to the shadow page directory entry for the specified
3987	* address, 32-bit.
3988	*
3989	* @returns Pointer to the shadow 32-bit PDE.
3990	* @param pPGM Pointer to the PGM instance data.
3991	* @param GCPtr The address.
3992	*/
3993	DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3994	{
3995	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
3996
3997	PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
3998	AssertReturn(pPde, NULL);
3999	return &pPde->a[iPd];
4000	}
4001
4002
4003	/**
4004	* Gets the shadow page pointer table, PAE.
4005	*
4006	* @returns Pointer to the shadow PAE PDPT.
4007	* @param pPGM Pointer to the PGM instance data.
4008	*/
4009	DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
4010	{
4011	return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4012	}
4013
4014
4015	/**
4016	* Gets the shadow page directory for the specified address, PAE.
4017	*
4018	* @returns Pointer to the shadow PD.
4019	* @param pPGM Pointer to the PGM instance data.
4020	* @param GCPtr The address.
4021	*/
4022	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4023	{
4024	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4025	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
4026
4027	if (!pPdpt->a[iPdpt].n.u1Present)
4028	return NULL;
4029
4030	/* Fetch the pgm pool shadow descriptor. */
4031	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4032	AssertReturn(pShwPde, NULL);
4033
4034	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4035	}
4036
4037
4038	/**
4039	* Gets the shadow page directory for the specified address, PAE.
4040	*
4041	* @returns Pointer to the shadow PD.
4042	* @param pPGM Pointer to the PGM instance data.
4043	* @param GCPtr The address.
4044	*/
4045	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
4046	{
4047	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4048
4049	if (!pPdpt->a[iPdpt].n.u1Present)
4050	return NULL;
4051
4052	/* Fetch the pgm pool shadow descriptor. */
4053	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4054	AssertReturn(pShwPde, NULL);
4055
4056	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4057	}
4058
4059
4060	/**
4061	* Gets the shadow page directory entry, PAE.
4062	*
4063	* @returns PDE.
4064	* @param pPGM Pointer to the PGM instance data.
4065	* @param GCPtr The address.
4066	*/
4067	DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4068	{
4069	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4070
4071	PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4072	if (!pShwPde)
4073	{
4074	X86PDEPAE ZeroPde = {0};
4075	return ZeroPde;
4076	}
4077	return pShwPde->a[iPd];
4078	}
4079
4080
4081	/**
4082	* Gets the pointer to the shadow page directory entry for an address, PAE.
4083	*
4084	* @returns Pointer to the PDE.
4085	* @param pPGM Pointer to the PGM instance data.
4086	* @param GCPtr The address.
4087	*/
4088	DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4089	{
4090	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4091
4092	PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4093	AssertReturn(pPde, NULL);
4094	return &pPde->a[iPd];
4095	}
4096
4097	#ifndef IN_RC
4098
4099	/**
4100	* Gets the shadow page map level-4 pointer.
4101	*
4102	* @returns Pointer to the shadow PML4.
4103	* @param pPGM Pointer to the PGM instance data.
4104	*/
4105	DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
4106	{
4107	return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4108	}
4109
4110
4111	/**
4112	* Gets the shadow page map level-4 entry for the specified address.
4113	*
4114	* @returns The entry.
4115	* @param pPGM Pointer to the PGM instance data.
4116	* @param GCPtr The address.
4117	*/
4118	DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
4119	{
4120	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4121	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4122
4123	if (!pShwPml4)
4124	{
4125	X86PML4E ZeroPml4e = {0};
4126	return ZeroPml4e;
4127	}
4128	return pShwPml4->a[iPml4];
4129	}
4130
4131
4132	/**
4133	* Gets the pointer to the specified shadow page map level-4 entry.
4134	*
4135	* @returns The entry.
4136	* @param pPGM Pointer to the PGM instance data.
4137	* @param iPml4 The PML4 index.
4138	*/
4139	DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4140	{
4141	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4142	if (!pShwPml4)
4143	return NULL;
4144	return &pShwPml4->a[iPml4];
4145	}
4146
4147
4148	/**
4149	* Gets the GUEST page directory pointer for the specified address.
4150	*
4151	* @returns The page directory in question.
4152	* @returns NULL if the page directory is not present or on an invalid page.
4153	* @param pPGM Pointer to the PGM instance data.
4154	* @param GCPtr The address.
4155	* @param piPD Receives the index into the returned page directory
4156	*/
4157	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
4158	{
4159	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4160	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4161	if (pGuestPml4->a[iPml4].n.u1Present)
4162	{
4163	PCX86PDPT pPdptTemp;
4164	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4165	AssertRCReturn(rc, NULL);
4166
4167	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4168	if (pPdptTemp->a[iPdpt].n.u1Present)
4169	{
4170	PX86PDPAE pPD;
4171	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4172	AssertRCReturn(rc, NULL);
4173
4174	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4175	return pPD;
4176	}
4177	}
4178	return NULL;
4179	}
4180
4181	#endif /* !IN_RC */
4182
4183	/**
4184	* Gets the page state for a physical handler.
4185	*
4186	* @returns The physical handler page state.
4187	* @param pCur The physical handler in question.
4188	*/
4189	DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
4190	{
4191	switch (pCur->enmType)
4192	{
4193	case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
4194	return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
4195
4196	case PGMPHYSHANDLERTYPE_MMIO:
4197	case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
4198	return PGM_PAGE_HNDL_PHYS_STATE_ALL;
4199
4200	default:
4201	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4202	}
4203	}
4204
4205
4206	/**
4207	* Gets the page state for a virtual handler.
4208	*
4209	* @returns The virtual handler page state.
4210	* @param pCur The virtual handler in question.
4211	* @remarks This should never be used on a hypervisor access handler.
4212	*/
4213	DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
4214	{
4215	switch (pCur->enmType)
4216	{
4217	case PGMVIRTHANDLERTYPE_WRITE:
4218	return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
4219	case PGMVIRTHANDLERTYPE_ALL:
4220	return PGM_PAGE_HNDL_VIRT_STATE_ALL;
4221	default:
4222	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4223	}
4224	}
4225
4226
4227	/**
4228	* Clears one physical page of a virtual handler
4229	*
4230	* @param pPGM Pointer to the PGM instance.
4231	* @param pCur Virtual handler structure
4232	* @param iPage Physical page index
4233	*
4234	* @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
4235	* need to care about other handlers in the same page.
4236	*/
4237	DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
4238	{
4239	const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
4240
4241	/*
4242	* Remove the node from the tree (it's supposed to be in the tree if we get here!).
4243	*/
4244	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4245	AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4246	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4247	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4248	#endif
4249	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
4250	{
4251	/* We're the head of the alias chain. */
4252	PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
4253	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4254	AssertReleaseMsg(pRemove != NULL,
4255	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4256	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4257	AssertReleaseMsg(pRemove == pPhys2Virt,
4258	("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4259	" got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4260	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
4261	pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
4262	#endif
4263	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4264	{
4265	/* Insert the next list in the alias chain into the tree. */
4266	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4267	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4268	AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4269	("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4270	pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
4271	#endif
4272	pNext->offNextAlias \|= PGMPHYS2VIRTHANDLER_IS_HEAD;
4273	bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
4274	AssertRelease(fRc);
4275	}
4276	}
4277	else
4278	{
4279	/* Locate the previous node in the alias chain. */
4280	PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
4281	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4282	AssertReleaseMsg(pPrev != pPhys2Virt,
4283	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4284	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4285	#endif
4286	for (;;)
4287	{
4288	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4289	if (pNext == pPhys2Virt)
4290	{
4291	/* unlink. */
4292	LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
4293	pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
4294	if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4295	pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
4296	else
4297	{
4298	PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4299	pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
4300	\| (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
4301	}
4302	break;
4303	}
4304
4305	/* next */
4306	if (pNext == pPrev)
4307	{
4308	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4309	AssertReleaseMsg(pNext != pPrev,
4310	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4311	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4312	#endif
4313	break;
4314	}
4315	pPrev = pNext;
4316	}
4317	}
4318	Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
4319	pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
4320	pPhys2Virt->offNextAlias = 0;
4321	pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
4322
4323	/*
4324	* Clear the ram flags for this page.
4325	*/
4326	PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
4327	AssertReturnVoid(pPage);
4328	PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
4329	}
4330
4331
4332	/**
4333	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4334	*
4335	* @returns Pointer to the shadow page structure.
4336	* @param pPool The pool.
4337	* @param HCPhys The HC physical address of the shadow page.
4338	*/
4339	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
4340	{
4341	/*
4342	* Look up the page.
4343	*/
4344	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
4345	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
4346	return pPage;
4347	}
4348
4349
4350	/**
4351	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4352	*
4353	* @returns Pointer to the shadow page structure.
4354	* @param pPool The pool.
4355	* @param idx The pool page index.
4356	*/
4357	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
4358	{
4359	AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
4360	return &pPool->aPages[idx];
4361	}
4362
4363
4364	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4365	/**
4366	* Clear references to guest physical memory.
4367	*
4368	* @param pPool The pool.
4369	* @param pPoolPage The pool page.
4370	* @param pPhysPage The physical guest page tracking structure.
4371	*/
4372	DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
4373	{
4374	/*
4375	* Just deal with the simple case here.
4376	*/
4377	# ifdef LOG_ENABLED
4378	const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
4379	# endif
4380	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4381	if (cRefs == 1)
4382	{
4383	Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
4384	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
4385	}
4386	else
4387	pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
4388	Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
4389	}
4390	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4391
4392
4393	#ifdef PGMPOOL_WITH_CACHE
4394	/**
4395	* Moves the page to the head of the age list.
4396	*
4397	* This is done when the cached page is used in one way or another.
4398	*
4399	* @param pPool The pool.
4400	* @param pPage The cached page.
4401	*/
4402	DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4403	{
4404	/*
4405	* Move to the head of the age list.
4406	*/
4407	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
4408	{
4409	/* unlink */
4410	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
4411	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
4412	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
4413	else
4414	pPool->iAgeTail = pPage->iAgePrev;
4415
4416	/* insert at head */
4417	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4418	pPage->iAgeNext = pPool->iAgeHead;
4419	Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
4420	pPool->iAgeHead = pPage->idx;
4421	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
4422	}
4423	}
4424	#endif /* PGMPOOL_WITH_CACHE */
4425
4426	/**
4427	* Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
4428	*
4429	* @param pVM VM Handle.
4430	* @param pPage PGM pool page
4431	*/
4432	DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4433	{
4434	Assert(!pPage->fLocked);
4435	pPage->fLocked = true;
4436	}
4437
4438
4439	/**
4440	* Unlocks a page to allow flushing again
4441	*
4442	* @param pVM VM Handle.
4443	* @param pPage PGM pool page
4444	*/
4445	DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4446	{
4447	Assert(pPage->fLocked);
4448	pPage->fLocked = false;
4449	}
4450
4451
4452	/**
4453	* Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
4454	*
4455	* @returns VBox status code.
4456	* @param pPage PGM pool page
4457	*/
4458	DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
4459	{
4460	if (pPage->fLocked)
4461	{
4462	LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
4463	if (pPage->cModifications)
4464	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
4465	return true;
4466	}
4467	return false;
4468	}
4469
4470	/**
4471	* Tells if mappings are to be put into the shadow page table or not
4472	*
4473	* @returns boolean result
4474	* @param pVM VM handle.
4475	*/
4476	DECLINLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
4477	{
4478	#ifdef IN_RING0
4479	/* There are no mappings in VT-x and AMD-V mode. */
4480	Assert(pPGM->fDisableMappings);
4481	return false;
4482	#else
4483	return !pPGM->fDisableMappings;
4484	#endif
4485	}
4486
4487	/** @} */
4488
4489	#endif
4490
4491

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMInternal.h@ 19488

Download in other formats: