PGMInternal.h@ 24911

Last change on this file since 24911 was 24874, checked in by vboxsync, 15 years ago
Main,VMM,VBoxManage: Added a parameter to IConsole::Teleport for specifying a max downtime and made PGM compare this against a rough estimate of the time it would take to deal with the rest of the pages.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 205.6 KB

Line
1	/* $Id: PGMInternal.h 24874 2009-11-23 15:37:58Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	#ifndef ___PGMInternal_h
23	#define ___PGMInternal_h
24
25	#include <VBox/cdefs.h>
26	#include <VBox/types.h>
27	#include <VBox/err.h>
28	#include <VBox/stam.h>
29	#include <VBox/param.h>
30	#include <VBox/vmm.h>
31	#include <VBox/mm.h>
32	#include <VBox/pdmcritsect.h>
33	#include <VBox/pdmapi.h>
34	#include <VBox/dis.h>
35	#include <VBox/dbgf.h>
36	#include <VBox/log.h>
37	#include <VBox/gmm.h>
38	#include <VBox/hwaccm.h>
39	#include <iprt/asm.h>
40	#include <iprt/assert.h>
41	#include <iprt/avl.h>
42	#include <iprt/critsect.h>
43	#include <iprt/sha.h>
44
45
46
47	/** @defgroup grp_pgm_int Internals
48	* @ingroup grp_pgm
49	* @internal
50	* @{
51	*/
52
53
54	/** @name PGM Compile Time Config
55	* @{
56	*/
57
58	/**
59	* Solve page is out of sync issues inside Guest Context (in PGMGC.cpp).
60	* Comment it if it will break something.
61	*/
62	#define PGM_OUT_OF_SYNC_IN_GC
63
64	/**
65	* Check and skip global PDEs for non-global flushes
66	*/
67	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
68
69	/**
70	* Optimization for PAE page tables that are modified often
71	*/
72	//#if 0 /* disabled again while debugging */
73	#ifndef IN_RC
74	# define PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
75	#endif
76	//#endif
77
78	/**
79	* Sync N pages instead of a whole page table
80	*/
81	#define PGM_SYNC_N_PAGES
82
83	/**
84	* Number of pages to sync during a page fault
85	*
86	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
87	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
88	*
89	* Note that \#PFs are much more expensive in the VT-x/AMD-V case due to
90	* world switch overhead, so let's sync more.
91	*/
92	# ifdef IN_RING0
93	/* Chose 32 based on the compile test in #4219; 64 shows worse stats.
94	* 32 again shows better results than 16; slightly more overhead in the \#PF handler,
95	* but ~5% fewer faults.
96	*/
97	# define PGM_SYNC_NR_PAGES 32
98	#else
99	# define PGM_SYNC_NR_PAGES 8
100	#endif
101
102	/**
103	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
104	*/
105	#define PGM_MAX_PHYSCACHE_ENTRIES 64
106	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
107
108	/** @def PGMPOOL_WITH_CACHE
109	* Enable agressive caching using the page pool.
110	*
111	* This requires PGMPOOL_WITH_USER_TRACKING and PGMPOOL_WITH_MONITORING.
112	*/
113	#define PGMPOOL_WITH_CACHE
114
115	/** @def PGMPOOL_WITH_MIXED_PT_CR3
116	* When defined, we'll deal with 'uncachable' pages.
117	*/
118	#ifdef PGMPOOL_WITH_CACHE
119	# define PGMPOOL_WITH_MIXED_PT_CR3
120	#endif
121
122	/** @def PGMPOOL_WITH_MONITORING
123	* Monitor the guest pages which are shadowed.
124	* When this is enabled, PGMPOOL_WITH_CACHE or PGMPOOL_WITH_GCPHYS_TRACKING must
125	* be enabled as well.
126	* @remark doesn't really work without caching now. (Mixed PT/CR3 change.)
127	*/
128	#ifdef PGMPOOL_WITH_CACHE
129	# define PGMPOOL_WITH_MONITORING
130	#endif
131
132	/** @def PGMPOOL_WITH_GCPHYS_TRACKING
133	* Tracking the of shadow pages mapping guest physical pages.
134	*
135	* This is very expensive, the current cache prototype is trying to figure out
136	* whether it will be acceptable with an agressive caching policy.
137	*/
138	#if defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
139	# define PGMPOOL_WITH_GCPHYS_TRACKING
140	#endif
141
142	/** @def PGMPOOL_WITH_USER_TRACKING
143	* Tracking users of shadow pages. This is required for the linking of shadow page
144	* tables and physical guest addresses.
145	*/
146	#if defined(PGMPOOL_WITH_GCPHYS_TRACKING) \|\| defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
147	# define PGMPOOL_WITH_USER_TRACKING
148	#endif
149
150	/** @def PGMPOOL_CFG_MAX_GROW
151	* The maximum number of pages to add to the pool in one go.
152	*/
153	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
154
155	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
156	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
157	*/
158	#ifdef VBOX_STRICT
159	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
160	#endif
161
162	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
163	* Enables the experimental lazy page allocation code. */
164	/#define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
165
166	/** @def VBOX_WITH_REAL_WRITE_MONITORED_PAGES
167	* Enables real write monitoring of pages, i.e. mapping them read-only and
168	* only making them writable when getting a write access #PF. */
169	#define VBOX_WITH_REAL_WRITE_MONITORED_PAGES
170
171	/** @} */
172
173
174	/** @name PDPT and PML4 flags.
175	* These are placed in the three bits available for system programs in
176	* the PDPT and PML4 entries.
177	* @{ */
178	/** The entry is a permanent one and it's must always be present.
179	* Never free such an entry. */
180	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
181	/** Mapping (hypervisor allocated pagetable). */
182	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
183	/** @} */
184
185	/** @name Page directory flags.
186	* These are placed in the three bits available for system programs in
187	* the page directory entries.
188	* @{ */
189	/** Mapping (hypervisor allocated pagetable). */
190	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
191	/** Made read-only to facilitate dirty bit tracking. */
192	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
193	/** @} */
194
195	/** @name Page flags.
196	* These are placed in the three bits available for system programs in
197	* the page entries.
198	* @{ */
199	/** Made read-only to facilitate dirty bit tracking. */
200	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
201
202	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
203	/** Scanned and approved by CSAM (tm).
204	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
205	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
206	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
207	#endif
208
209	/** @} */
210
211	/** @name Defines used to indicate the shadow and guest paging in the templates.
212	* @{ */
213	#define PGM_TYPE_REAL 1
214	#define PGM_TYPE_PROT 2
215	#define PGM_TYPE_32BIT 3
216	#define PGM_TYPE_PAE 4
217	#define PGM_TYPE_AMD64 5
218	#define PGM_TYPE_NESTED 6
219	#define PGM_TYPE_EPT 7
220	#define PGM_TYPE_MAX PGM_TYPE_EPT
221	/** @} */
222
223	/** Macro for checking if the guest is using paging.
224	* @param uGstType PGM_TYPE_*
225	* @param uShwType PGM_TYPE_*
226	* @remark ASSUMES certain order of the PGM_TYPE_* values.
227	*/
228	#define PGM_WITH_PAGING(uGstType, uShwType) \
229	( (uGstType) >= PGM_TYPE_32BIT \
230	&& (uShwType) != PGM_TYPE_NESTED \
231	&& (uShwType) != PGM_TYPE_EPT)
232
233	/** Macro for checking if the guest supports the NX bit.
234	* @param uGstType PGM_TYPE_*
235	* @param uShwType PGM_TYPE_*
236	* @remark ASSUMES certain order of the PGM_TYPE_* values.
237	*/
238	#define PGM_WITH_NX(uGstType, uShwType) \
239	( (uGstType) >= PGM_TYPE_PAE \
240	&& (uShwType) != PGM_TYPE_NESTED \
241	&& (uShwType) != PGM_TYPE_EPT)
242
243
244	/** @def PGM_HCPHYS_2_PTR
245	* Maps a HC physical page pool address to a virtual address.
246	*
247	* @returns VBox status code.
248	* @param pVM The VM handle.
249	* @param HCPhys The HC physical address to map to a virtual one.
250	* @param ppv Where to store the virtual address. No need to cast this.
251	*
252	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
253	* small page window employeed by that function. Be careful.
254	* @remark There is no need to assert on the result.
255	*/
256	#ifdef IN_RC
257	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
258	PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
259	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
260	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
261	pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
262	#else
263	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
264	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
265	#endif
266
267	/** @def PGM_HCPHYS_2_PTR_BY_PGM
268	* Maps a HC physical page pool address to a virtual address.
269	*
270	* @returns VBox status code.
271	* @param pPGM The PGM instance data.
272	* @param HCPhys The HC physical address to map to a virtual one.
273	* @param ppv Where to store the virtual address. No need to cast this.
274	*
275	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
276	* small page window employeed by that function. Be careful.
277	* @remark There is no need to assert on the result.
278	*/
279	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
280	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
281	pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
282	#else
283	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
284	PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
285	#endif
286
287	/** @def PGM_GCPHYS_2_PTR
288	* Maps a GC physical page address to a virtual address.
289	*
290	* @returns VBox status code.
291	* @param pVM The VM handle.
292	* @param GCPhys The GC physical address to map to a virtual one.
293	* @param ppv Where to store the virtual address. No need to cast this.
294	*
295	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
296	* small page window employeed by that function. Be careful.
297	* @remark There is no need to assert on the result.
298	*/
299	#ifdef IN_RC
300	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
301	PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
302	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
303	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
304	pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
305	#else
306	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
307	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
308	#endif
309
310	/** @def PGM_GCPHYS_2_PTR_BY_PGMCPU
311	* Maps a GC physical page address to a virtual address.
312	*
313	* @returns VBox status code.
314	* @param pPGM Pointer to the PGM instance data.
315	* @param GCPhys The GC physical address to map to a virtual one.
316	* @param ppv Where to store the virtual address. No need to cast this.
317	*
318	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
319	* small page window employeed by that function. Be careful.
320	* @remark There is no need to assert on the result.
321	*/
322	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
323	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
324	pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), GCPhys, (void **)(ppv))
325	#else
326	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
327	PGM_GCPHYS_2_PTR(PGMCPU2VM(pPGM), GCPhys, ppv)
328	#endif
329
330	/** @def PGM_GCPHYS_2_PTR_EX
331	* Maps a unaligned GC physical page address to a virtual address.
332	*
333	* @returns VBox status code.
334	* @param pVM The VM handle.
335	* @param GCPhys The GC physical address to map to a virtual one.
336	* @param ppv Where to store the virtual address. No need to cast this.
337	*
338	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
339	* small page window employeed by that function. Be careful.
340	* @remark There is no need to assert on the result.
341	*/
342	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
343	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
344	PGMDynMapGCPageOff(pVM, GCPhys, (void **)(ppv))
345	#else
346	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
347	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
348	#endif
349
350	/** @def PGM_INVL_PG
351	* Invalidates a page.
352	*
353	* @param pVCpu The VMCPU handle.
354	* @param GCVirt The virtual address of the page to invalidate.
355	*/
356	#ifdef IN_RC
357	# define PGM_INVL_PG(pVCpu, GCVirt) ASMInvalidatePage((void *)(GCVirt))
358	#elif defined(IN_RING0)
359	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
360	#else
361	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
362	#endif
363
364	/** @def PGM_INVL_PG_ALL_VCPU
365	* Invalidates a page on all VCPUs
366	*
367	* @param pVM The VM handle.
368	* @param GCVirt The virtual address of the page to invalidate.
369	*/
370	#ifdef IN_RC
371	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) ASMInvalidatePage((void *)(GCVirt))
372	#elif defined(IN_RING0)
373	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
374	#else
375	# define PGM_INVL_PG_ALL_VCPU(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
376	#endif
377
378	/** @def PGM_INVL_BIG_PG
379	* Invalidates a 4MB page directory entry.
380	*
381	* @param pVCpu The VMCPU handle.
382	* @param GCVirt The virtual address within the page directory to invalidate.
383	*/
384	#ifdef IN_RC
385	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) ASMReloadCR3()
386	#elif defined(IN_RING0)
387	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
388	#else
389	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
390	#endif
391
392	/** @def PGM_INVL_VCPU_TLBS()
393	* Invalidates the TLBs of the specified VCPU
394	*
395	* @param pVCpu The VMCPU handle.
396	*/
397	#ifdef IN_RC
398	# define PGM_INVL_VCPU_TLBS(pVCpu) ASMReloadCR3()
399	#elif defined(IN_RING0)
400	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
401	#else
402	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
403	#endif
404
405	/** @def PGM_INVL_ALL_VCPU_TLBS()
406	* Invalidates the TLBs of all VCPUs
407	*
408	* @param pVM The VM handle.
409	*/
410	#ifdef IN_RC
411	# define PGM_INVL_ALL_VCPU_TLBS(pVM) ASMReloadCR3()
412	#elif defined(IN_RING0)
413	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
414	#else
415	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
416	#endif
417
418	/** Size of the GCPtrConflict array in PGMMAPPING.
419	* @remarks Must be a power of two. */
420	#define PGMMAPPING_CONFLICT_MAX 8
421
422	/**
423	* Structure for tracking GC Mappings.
424	*
425	* This structure is used by linked list in both GC and HC.
426	*/
427	typedef struct PGMMAPPING
428	{
429	/** Pointer to next entry. */
430	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
431	/** Pointer to next entry. */
432	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
433	/** Pointer to next entry. */
434	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
435	/** Indicate whether this entry is finalized. */
436	bool fFinalized;
437	/** Start Virtual address. */
438	RTGCPTR GCPtr;
439	/** Last Virtual address (inclusive). */
440	RTGCPTR GCPtrLast;
441	/** Range size (bytes). */
442	RTGCPTR cb;
443	/** Pointer to relocation callback function. */
444	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
445	/** User argument to the callback. */
446	R3PTRTYPE(void *) pvUser;
447	/** Mapping description / name. For easing debugging. */
448	R3PTRTYPE(const char *) pszDesc;
449	/** Last 8 addresses that caused conflicts. */
450	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
451	/** Number of conflicts for this hypervisor mapping. */
452	uint32_t cConflicts;
453	/** Number of page tables. */
454	uint32_t cPTs;
455
456	/** Array of page table mapping data. Each entry
457	* describes one page table. The array can be longer
458	* than the declared length.
459	*/
460	struct
461	{
462	/** The HC physical address of the page table. */
463	RTHCPHYS HCPhysPT;
464	/** The HC physical address of the first PAE page table. */
465	RTHCPHYS HCPhysPaePT0;
466	/** The HC physical address of the second PAE page table. */
467	RTHCPHYS HCPhysPaePT1;
468	/** The HC virtual address of the 32-bit page table. */
469	R3PTRTYPE(PX86PT) pPTR3;
470	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
471	R3PTRTYPE(PX86PTPAE) paPaePTsR3;
472	/** The RC virtual address of the 32-bit page table. */
473	RCPTRTYPE(PX86PT) pPTRC;
474	/** The RC virtual address of the two PAE page table. */
475	RCPTRTYPE(PX86PTPAE) paPaePTsRC;
476	/** The R0 virtual address of the 32-bit page table. */
477	R0PTRTYPE(PX86PT) pPTR0;
478	/** The R0 virtual address of the two PAE page table. */
479	R0PTRTYPE(PX86PTPAE) paPaePTsR0;
480	} aPTs[1];
481	} PGMMAPPING;
482	/** Pointer to structure for tracking GC Mappings. */
483	typedef struct PGMMAPPING *PPGMMAPPING;
484
485
486	/**
487	* Physical page access handler structure.
488	*
489	* This is used to keep track of physical address ranges
490	* which are being monitored in some kind of way.
491	*/
492	typedef struct PGMPHYSHANDLER
493	{
494	AVLROGCPHYSNODECORE Core;
495	/** Access type. */
496	PGMPHYSHANDLERTYPE enmType;
497	/** Number of pages to update. */
498	uint32_t cPages;
499	/** Pointer to R3 callback function. */
500	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
501	/** User argument for R3 handlers. */
502	R3PTRTYPE(void *) pvUserR3;
503	/** Pointer to R0 callback function. */
504	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
505	/** User argument for R0 handlers. */
506	R0PTRTYPE(void *) pvUserR0;
507	/** Pointer to RC callback function. */
508	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
509	/** User argument for RC handlers. */
510	RCPTRTYPE(void *) pvUserRC;
511	/** Description / Name. For easing debugging. */
512	R3PTRTYPE(const char *) pszDesc;
513	#ifdef VBOX_WITH_STATISTICS
514	/** Profiling of this handler. */
515	STAMPROFILE Stat;
516	#endif
517	} PGMPHYSHANDLER;
518	/** Pointer to a physical page access handler structure. */
519	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
520
521
522	/**
523	* Cache node for the physical addresses covered by a virtual handler.
524	*/
525	typedef struct PGMPHYS2VIRTHANDLER
526	{
527	/** Core node for the tree based on physical ranges. */
528	AVLROGCPHYSNODECORE Core;
529	/** Offset from this struct to the PGMVIRTHANDLER structure. */
530	int32_t offVirtHandler;
531	/** Offset of the next alias relative to this one.
532	* Bit 0 is used for indicating whether we're in the tree.
533	* Bit 1 is used for indicating that we're the head node.
534	*/
535	int32_t offNextAlias;
536	} PGMPHYS2VIRTHANDLER;
537	/** Pointer to a phys to virtual handler structure. */
538	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
539
540	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
541	* node is in the tree. */
542	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
543	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
544	* node is in the head of an alias chain.
545	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
546	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
547	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
548	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
549
550
551	/**
552	* Virtual page access handler structure.
553	*
554	* This is used to keep track of virtual address ranges
555	* which are being monitored in some kind of way.
556	*/
557	typedef struct PGMVIRTHANDLER
558	{
559	/** Core node for the tree based on virtual ranges. */
560	AVLROGCPTRNODECORE Core;
561	/** Size of the range (in bytes). */
562	RTGCPTR cb;
563	/** Number of cache pages. */
564	uint32_t cPages;
565	/** Access type. */
566	PGMVIRTHANDLERTYPE enmType;
567	/** Pointer to the RC callback function. */
568	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
569	#if HC_ARCH_BITS == 64
570	RTRCPTR padding;
571	#endif
572	/** Pointer to the R3 callback function for invalidation. */
573	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
574	/** Pointer to the R3 callback function. */
575	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
576	/** Description / Name. For easing debugging. */
577	R3PTRTYPE(const char *) pszDesc;
578	#ifdef VBOX_WITH_STATISTICS
579	/** Profiling of this handler. */
580	STAMPROFILE Stat;
581	#endif
582	/** Array of cached physical addresses for the monitored ranged. */
583	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
584	} PGMVIRTHANDLER;
585	/** Pointer to a virtual page access handler structure. */
586	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
587
588
589	/**
590	* Page type.
591	*
592	* @remarks This enum has to fit in a 3-bit field (see PGMPAGE::u3Type).
593	* @remarks This is used in the saved state, so changes to it requires bumping
594	* the saved state version.
595	* @todo So, convert to \#defines!
596	*/
597	typedef enum PGMPAGETYPE
598	{
599	/** The usual invalid zero entry. */
600	PGMPAGETYPE_INVALID = 0,
601	/** RAM page. (RWX) */
602	PGMPAGETYPE_RAM,
603	/** MMIO2 page. (RWX) */
604	PGMPAGETYPE_MMIO2,
605	/** MMIO2 page aliased over an MMIO page. (RWX)
606	* See PGMHandlerPhysicalPageAlias(). */
607	PGMPAGETYPE_MMIO2_ALIAS_MMIO,
608	/** Shadowed ROM. (RWX) */
609	PGMPAGETYPE_ROM_SHADOW,
610	/** ROM page. (R-X) */
611	PGMPAGETYPE_ROM,
612	/** MMIO page. (---) */
613	PGMPAGETYPE_MMIO,
614	/** End of valid entries. */
615	PGMPAGETYPE_END
616	} PGMPAGETYPE;
617	AssertCompile(PGMPAGETYPE_END <= 7);
618
619	/** @name Page type predicates.
620	* @{ */
621	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
622	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
623	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
624	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
625	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
626	/** @} */
627
628
629	/**
630	* A Physical Guest Page tracking structure.
631	*
632	* The format of this structure is complicated because we have to fit a lot
633	* of information into as few bits as possible. The format is also subject
634	* to change (there is one comming up soon). Which means that for we'll be
635	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
636	* accesses to the structure.
637	*/
638	typedef struct PGMPAGE
639	{
640	/** The physical address and the Page ID. */
641	RTHCPHYS HCPhysAndPageID;
642	/** Combination of:
643	* - [0-7]: u2HandlerPhysStateY - the physical handler state
644	* (PGM_PAGE_HNDL_PHYS_STATE_*).
645	* - [8-9]: u2HandlerVirtStateY - the virtual handler state
646	* (PGM_PAGE_HNDL_VIRT_STATE_*).
647	* - [15]: fWrittenToY - flag indicating that a write monitored page was
648	* written to when set.
649	* - [10-14]: 5 unused bits.
650	* @remarks Warning! All accesses to the bits are hardcoded.
651	*
652	* @todo Change this to a union with both bitfields, u8 and u accessors.
653	* That'll help deal with some of the hardcoded accesses.
654	*
655	* @todo Include uStateY and uTypeY as well so it becomes 32-bit. This
656	* will make it possible to turn some of the 16-bit accesses into
657	* 32-bit ones, which may be efficient (stalls).
658	*/
659	RTUINT16U u16MiscY;
660	/** The page state.
661	* Only 2 bits are really needed for this. */
662	uint8_t uStateY;
663	/** The page type (PGMPAGETYPE).
664	* Only 3 bits are really needed for this. */
665	uint8_t uTypeY;
666	/** Usage tracking (page pool). */
667	uint16_t u16TrackingY;
668	/** The number of read locks on this page. */
669	uint8_t cReadLocksY;
670	/** The number of write locks on this page. */
671	uint8_t cWriteLocksY;
672	} PGMPAGE;
673	AssertCompileSize(PGMPAGE, 16);
674	/** Pointer to a physical guest page. */
675	typedef PGMPAGE *PPGMPAGE;
676	/** Pointer to a const physical guest page. */
677	typedef const PGMPAGE *PCPGMPAGE;
678	/** Pointer to a physical guest page pointer. */
679	typedef PPGMPAGE *PPPGMPAGE;
680
681
682	/**
683	* Clears the page structure.
684	* @param pPage Pointer to the physical guest page tracking structure.
685	*/
686	#define PGM_PAGE_CLEAR(pPage) \
687	do { \
688	(pPage)->HCPhysAndPageID = 0; \
689	(pPage)->uStateY = 0; \
690	(pPage)->uTypeY = 0; \
691	(pPage)->u16MiscY.u = 0; \
692	(pPage)->u16TrackingY = 0; \
693	(pPage)->cReadLocksY = 0; \
694	(pPage)->cWriteLocksY = 0; \
695	} while (0)
696
697	/**
698	* Initializes the page structure.
699	* @param pPage Pointer to the physical guest page tracking structure.
700	*/
701	#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
702	do { \
703	RTHCPHYS SetHCPhysTmp = (_HCPhys); \
704	AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
705	(pPage)->HCPhysAndPageID = (SetHCPhysTmp << (28-12)) \| ((_idPage) & UINT32_C(0x0fffffff)); \
706	(pPage)->uStateY = (_uState); \
707	(pPage)->uTypeY = (_uType); \
708	(pPage)->u16MiscY.u = 0; \
709	(pPage)->u16TrackingY = 0; \
710	(pPage)->cReadLocksY = 0; \
711	(pPage)->cWriteLocksY = 0; \
712	} while (0)
713
714	/**
715	* Initializes the page structure of a ZERO page.
716	* @param pPage Pointer to the physical guest page tracking structure.
717	* @param pVM The VM handle (for getting the zero page address).
718	* @param uType The page type (PGMPAGETYPE).
719	*/
720	#define PGM_PAGE_INIT_ZERO(pPage, pVM, uType) \
721	PGM_PAGE_INIT((pPage), (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (uType), PGM_PAGE_STATE_ZERO)
722
723
724	/** @name The Page state, PGMPAGE::uStateY.
725	* @{ */
726	/** The zero page.
727	* This is a per-VM page that's never ever mapped writable. */
728	#define PGM_PAGE_STATE_ZERO 0
729	/** A allocated page.
730	* This is a per-VM page allocated from the page pool (or wherever
731	* we get MMIO2 pages from if the type is MMIO2).
732	*/
733	#define PGM_PAGE_STATE_ALLOCATED 1
734	/** A allocated page that's being monitored for writes.
735	* The shadow page table mappings are read-only. When a write occurs, the
736	* fWrittenTo member is set, the page remapped as read-write and the state
737	* moved back to allocated. */
738	#define PGM_PAGE_STATE_WRITE_MONITORED 2
739	/** The page is shared, aka. copy-on-write.
740	* This is a page that's shared with other VMs. */
741	#define PGM_PAGE_STATE_SHARED 3
742	/** @} */
743
744
745	/**
746	* Gets the page state.
747	* @returns page state (PGM_PAGE_STATE_*).
748	* @param pPage Pointer to the physical guest page tracking structure.
749	*/
750	#define PGM_PAGE_GET_STATE(pPage) ( (pPage)->uStateY )
751
752	/**
753	* Sets the page state.
754	* @param pPage Pointer to the physical guest page tracking structure.
755	* @param _uState The new page state.
756	*/
757	#define PGM_PAGE_SET_STATE(pPage, _uState) do { (pPage)->uStateY = (_uState); } while (0)
758
759
760	/**
761	* Gets the host physical address of the guest page.
762	* @returns host physical address (RTHCPHYS).
763	* @param pPage Pointer to the physical guest page tracking structure.
764	*/
765	#define PGM_PAGE_GET_HCPHYS(pPage) ( ((pPage)->HCPhysAndPageID >> 28) << 12 )
766
767	/**
768	* Sets the host physical address of the guest page.
769	* @param pPage Pointer to the physical guest page tracking structure.
770	* @param _HCPhys The new host physical address.
771	*/
772	#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
773	do { \
774	RTHCPHYS SetHCPhysTmp = (_HCPhys); \
775	AssertFatal(!(SetHCPhysTmp & ~UINT64_C(0x0000fffffffff000))); \
776	(pPage)->HCPhysAndPageID = ((pPage)->HCPhysAndPageID & UINT32_C(0x0fffffff)) \
777	\| (SetHCPhysTmp << (28-12)); \
778	} while (0)
779
780	/**
781	* Get the Page ID.
782	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
783	* @param pPage Pointer to the physical guest page tracking structure.
784	*/
785	#define PGM_PAGE_GET_PAGEID(pPage) ( (uint32_t)((pPage)->HCPhysAndPageID & UINT32_C(0x0fffffff)) )
786
787	/**
788	* Sets the Page ID.
789	* @param pPage Pointer to the physical guest page tracking structure.
790	*/
791	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) \
792	do { \
793	(pPage)->HCPhysAndPageID = (((pPage)->HCPhysAndPageID) & UINT64_C(0xfffffffff0000000)) \
794	\| ((_idPage) & UINT32_C(0x0fffffff)); \
795	} while (0)
796
797	/**
798	* Get the Chunk ID.
799	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
800	* @param pPage Pointer to the physical guest page tracking structure.
801	*/
802	#define PGM_PAGE_GET_CHUNKID(pPage) ( PGM_PAGE_GET_PAGEID(pPage) >> GMM_CHUNKID_SHIFT )
803
804	/**
805	* Get the index of the page within the allocation chunk.
806	* @returns The page index.
807	* @param pPage Pointer to the physical guest page tracking structure.
808	*/
809	#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysAndPageID & GMM_PAGEID_IDX_MASK) )
810
811	/**
812	* Gets the page type.
813	* @returns The page type.
814	* @param pPage Pointer to the physical guest page tracking structure.
815	*/
816	#define PGM_PAGE_GET_TYPE(pPage) (pPage)->uTypeY
817
818	/**
819	* Sets the page type.
820	* @param pPage Pointer to the physical guest page tracking structure.
821	* @param _enmType The new page type (PGMPAGETYPE).
822	*/
823	#define PGM_PAGE_SET_TYPE(pPage, _enmType) do { (pPage)->uTypeY = (_enmType); } while (0)
824
825	/**
826	* Checks if the page is marked for MMIO.
827	* @returns true/false.
828	* @param pPage Pointer to the physical guest page tracking structure.
829	*/
830	#define PGM_PAGE_IS_MMIO(pPage) ( (pPage)->uTypeY == PGMPAGETYPE_MMIO )
831
832	/**
833	* Checks if the page is backed by the ZERO page.
834	* @returns true/false.
835	* @param pPage Pointer to the physical guest page tracking structure.
836	*/
837	#define PGM_PAGE_IS_ZERO(pPage) ( (pPage)->uStateY == PGM_PAGE_STATE_ZERO )
838
839	/**
840	* Checks if the page is backed by a SHARED page.
841	* @returns true/false.
842	* @param pPage Pointer to the physical guest page tracking structure.
843	*/
844	#define PGM_PAGE_IS_SHARED(pPage) ( (pPage)->uStateY == PGM_PAGE_STATE_SHARED )
845
846
847	/**
848	* Marks the paget as written to (for GMM change monitoring).
849	* @param pPage Pointer to the physical guest page tracking structure.
850	*/
851	#define PGM_PAGE_SET_WRITTEN_TO(pPage) do { (pPage)->u16MiscY.au8[1] \|= UINT8_C(0x80); } while (0)
852
853	/**
854	* Clears the written-to indicator.
855	* @param pPage Pointer to the physical guest page tracking structure.
856	*/
857	#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage) do { (pPage)->u16MiscY.au8[1] &= UINT8_C(0x7f); } while (0)
858
859	/**
860	* Checks if the page was marked as written-to.
861	* @returns true/false.
862	* @param pPage Pointer to the physical guest page tracking structure.
863	*/
864	#define PGM_PAGE_IS_WRITTEN_TO(pPage) ( !!((pPage)->u16MiscY.au8[1] & UINT8_C(0x80)) )
865
866
867	/** Enabled optimized access handler tests.
868	* These optimizations makes ASSUMPTIONS about the state values and the u16MiscY
869	* layout. When enabled, the compiler should normally generate more compact
870	* code.
871	*/
872	#define PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS 1
873
874	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateY).
875	*
876	* @remarks The values are assigned in order of priority, so we can calculate
877	* the correct state for a page with different handlers installed.
878	* @{ */
879	/** No handler installed. */
880	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
881	/** Monitoring is temporarily disabled. */
882	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
883	/** Write access is monitored. */
884	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
885	/** All access is monitored. */
886	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
887	/** @} */
888
889	/**
890	* Gets the physical access handler state of a page.
891	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
892	* @param pPage Pointer to the physical guest page tracking structure.
893	*/
894	#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) \
895	( (pPage)->u16MiscY.au8[0] )
896
897	/**
898	* Sets the physical access handler state of a page.
899	* @param pPage Pointer to the physical guest page tracking structure.
900	* @param _uState The new state value.
901	*/
902	#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
903	do { (pPage)->u16MiscY.au8[0] = (_uState); } while (0)
904
905	/**
906	* Checks if the page has any physical access handlers, including temporariliy disabled ones.
907	* @returns true/false
908	* @param pPage Pointer to the physical guest page tracking structure.
909	*/
910	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) \
911	( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE )
912
913	/**
914	* Checks if the page has any active physical access handlers.
915	* @returns true/false
916	* @param pPage Pointer to the physical guest page tracking structure.
917	*/
918	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) \
919	( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
920
921
922	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateY).
923	*
924	* @remarks The values are assigned in order of priority, so we can calculate
925	* the correct state for a page with different handlers installed.
926	* @{ */
927	/** No handler installed. */
928	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
929	/* 1 is reserved so the lineup is identical with the physical ones. */
930	/** Write access is monitored. */
931	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
932	/** All access is monitored. */
933	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
934	/** @} */
935
936	/**
937	* Gets the virtual access handler state of a page.
938	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
939	* @param pPage Pointer to the physical guest page tracking structure.
940	*/
941	#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ( (pPage)->u16MiscY.au8[1] & UINT8_C(0x03) )
942
943	/**
944	* Sets the virtual access handler state of a page.
945	* @param pPage Pointer to the physical guest page tracking structure.
946	* @param _uState The new state value.
947	*/
948	#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
949	do { \
950	(pPage)->u16MiscY.au8[1] = ((pPage)->u16MiscY.au8[1] & UINT8_C(0xfc)) \
951	\| ((_uState) & UINT8_C(0x03)); \
952	} while (0)
953
954	/**
955	* Checks if the page has any virtual access handlers.
956	* @returns true/false
957	* @param pPage Pointer to the physical guest page tracking structure.
958	*/
959	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) \
960	( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
961
962	/**
963	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
964	* virtual handlers.
965	* @returns true/false
966	* @param pPage Pointer to the physical guest page tracking structure.
967	*/
968	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) \
969	PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
970
971
972	/**
973	* Checks if the page has any access handlers, including temporarily disabled ones.
974	* @returns true/false
975	* @param pPage Pointer to the physical guest page tracking structure.
976	*/
977	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
978	# define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
979	( ((pPage)->u16MiscY.u & UINT16_C(0x0303)) != 0 )
980	#else
981	# define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
982	( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE \
983	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) != PGM_PAGE_HNDL_VIRT_STATE_NONE )
984	#endif
985
986	/**
987	* Checks if the page has any active access handlers.
988	* @returns true/false
989	* @param pPage Pointer to the physical guest page tracking structure.
990	*/
991	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
992	# define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
993	( ((pPage)->u16MiscY.u & UINT16_C(0x0202)) != 0 )
994	#else
995	# define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
996	( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
997	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
998	#endif
999
1000	/**
1001	* Checks if the page has any active access handlers catching all accesses.
1002	* @returns true/false
1003	* @param pPage Pointer to the physical guest page tracking structure.
1004	*/
1005	#ifdef PGM_PAGE_WITH_OPTIMIZED_HANDLER_ACCESS
1006	# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
1007	( ( ((pPage)->u16MiscY.au8[0] \| (pPage)->u16MiscY.au8[1]) & UINT8_C(0x3) ) \
1008	== PGM_PAGE_HNDL_PHYS_STATE_ALL )
1009	#else
1010	# define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
1011	( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL \
1012	\|\| PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) == PGM_PAGE_HNDL_VIRT_STATE_ALL )
1013	#endif
1014
1015
1016	/** @def PGM_PAGE_GET_TRACKING
1017	* Gets the packed shadow page pool tracking data associated with a guest page.
1018	* @returns uint16_t containing the data.
1019	* @param pPage Pointer to the physical guest page tracking structure.
1020	*/
1021	#define PGM_PAGE_GET_TRACKING(pPage) ( (pPage)->u16TrackingY )
1022
1023	/** @def PGM_PAGE_SET_TRACKING
1024	* Sets the packed shadow page pool tracking data associated with a guest page.
1025	* @param pPage Pointer to the physical guest page tracking structure.
1026	* @param u16TrackingData The tracking data to store.
1027	*/
1028	#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
1029	do { (pPage)->u16TrackingY = (u16TrackingData); } while (0)
1030
1031	/** @def PGM_PAGE_GET_TD_CREFS
1032	* Gets the @a cRefs tracking data member.
1033	* @returns cRefs.
1034	* @param pPage Pointer to the physical guest page tracking structure.
1035	*/
1036	#define PGM_PAGE_GET_TD_CREFS(pPage) \
1037	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1038
1039	/** @def PGM_PAGE_GET_TD_IDX
1040	* Gets the @a idx tracking data member.
1041	* @returns idx.
1042	* @param pPage Pointer to the physical guest page tracking structure.
1043	*/
1044	#define PGM_PAGE_GET_TD_IDX(pPage) \
1045	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1046
1047
1048	/** Max number of locks on a page. */
1049	#define PGM_PAGE_MAX_LOCKS UINT8_C(254)
1050
1051	/** Get the read lock count.
1052	* @returns count.
1053	* @param pPage Pointer to the physical guest page tracking structure.
1054	*/
1055	#define PGM_PAGE_GET_READ_LOCKS(pPage) ( (pPage)->cReadLocksY )
1056
1057	/** Get the write lock count.
1058	* @returns count.
1059	* @param pPage Pointer to the physical guest page tracking structure.
1060	*/
1061	#define PGM_PAGE_GET_WRITE_LOCKS(pPage) ( (pPage)->cWriteLocksY )
1062
1063	/** Decrement the read lock counter.
1064	* @param pPage Pointer to the physical guest page tracking structure.
1065	*/
1066	#define PGM_PAGE_DEC_READ_LOCKS(pPage) do { --(pPage)->cReadLocksY; } while (0)
1067
1068	/** Decrement the write lock counter.
1069	* @param pPage Pointer to the physical guest page tracking structure.
1070	*/
1071	#define PGM_PAGE_DEC_WRITE_LOCKS(pPage) do { --(pPage)->cWriteLocksY; } while (0)
1072
1073	/** Increment the read lock counter.
1074	* @param pPage Pointer to the physical guest page tracking structure.
1075	*/
1076	#define PGM_PAGE_INC_READ_LOCKS(pPage) do { ++(pPage)->cReadLocksY; } while (0)
1077
1078	/** Increment the write lock counter.
1079	* @param pPage Pointer to the physical guest page tracking structure.
1080	*/
1081	#define PGM_PAGE_INC_WRITE_LOCKS(pPage) do { ++(pPage)->cWriteLocksY; } while (0)
1082
1083
1084	#if 0
1085	/** Enables sanity checking of write monitoring using CRC-32. */
1086	# define PGMLIVESAVERAMPAGE_WITH_CRC32
1087	#endif
1088
1089	/**
1090	* Per page live save tracking data.
1091	*/
1092	typedef struct PGMLIVESAVERAMPAGE
1093	{
1094	/** Number of times it has been dirtied. */
1095	uint32_t cDirtied : 24;
1096	/** Whether it is currently dirty. */
1097	uint32_t fDirty : 1;
1098	/** Ignore the page.
1099	* This is used for pages that has been MMIO, MMIO2 or ROM pages once. We will
1100	* deal with these after pausing the VM and DevPCI have said it bit about
1101	* remappings. */
1102	uint32_t fIgnore : 1;
1103	/** Was a ZERO page last time around. */
1104	uint32_t fZero : 1;
1105	/** Was a SHARED page last time around. */
1106	uint32_t fShared : 1;
1107	/** Whether the page is/was write monitored in a previous pass. */
1108	uint32_t fWriteMonitored : 1;
1109	/** Whether the page is/was write monitored earlier in this pass. */
1110	uint32_t fWriteMonitoredJustNow : 1;
1111	/** Bits reserved for future use. */
1112	uint32_t u2Reserved : 2;
1113	#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1114	/** CRC-32 for the page. This is for internal consistency checks. */
1115	uint32_t u32Crc;
1116	#endif
1117	} PGMLIVESAVERAMPAGE;
1118	#ifdef PGMLIVESAVERAMPAGE_WITH_CRC32
1119	AssertCompileSize(PGMLIVESAVERAMPAGE, 8);
1120	#else
1121	AssertCompileSize(PGMLIVESAVERAMPAGE, 4);
1122	#endif
1123	/** Pointer to the per page live save tracking data. */
1124	typedef PGMLIVESAVERAMPAGE *PPGMLIVESAVERAMPAGE;
1125
1126	/** The max value of PGMLIVESAVERAMPAGE::cDirtied. */
1127	#define PGMLIVSAVEPAGE_MAX_DIRTIED 0x00fffff0
1128
1129
1130	/**
1131	* Ram range for GC Phys to HC Phys conversion.
1132	*
1133	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
1134	* conversions too, but we'll let MM handle that for now.
1135	*
1136	* This structure is used by linked lists in both GC and HC.
1137	*/
1138	typedef struct PGMRAMRANGE
1139	{
1140	/** Start of the range. Page aligned. */
1141	RTGCPHYS GCPhys;
1142	/** Size of the range. (Page aligned of course). */
1143	RTGCPHYS cb;
1144	/** Pointer to the next RAM range - for R3. */
1145	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
1146	/** Pointer to the next RAM range - for R0. */
1147	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
1148	/** Pointer to the next RAM range - for RC. */
1149	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
1150	/** PGM_RAM_RANGE_FLAGS_* flags. */
1151	uint32_t fFlags;
1152	/** Last address in the range (inclusive). Page aligned (-1). */
1153	RTGCPHYS GCPhysLast;
1154	/** Start of the HC mapping of the range. This is only used for MMIO2. */
1155	R3PTRTYPE(void *) pvR3;
1156	/** Live save per page tracking data. */
1157	R3PTRTYPE(PPGMLIVESAVERAMPAGE) paLSPages;
1158	/** The range description. */
1159	R3PTRTYPE(const char *) pszDesc;
1160	/** Pointer to self - R0 pointer. */
1161	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1162	/** Pointer to self - RC pointer. */
1163	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1164	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1165	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 1 : 3];
1166	/** Array of physical guest page tracking structures. */
1167	PGMPAGE aPages[1];
1168	} PGMRAMRANGE;
1169	/** Pointer to Ram range for GC Phys to HC Phys conversion. */
1170	typedef PGMRAMRANGE *PPGMRAMRANGE;
1171
1172	/** @name PGMRAMRANGE::fFlags
1173	* @{ */
1174	/** The RAM range is floating around as an independent guest mapping. */
1175	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1176	/** Ad hoc RAM range for an ROM mapping. */
1177	#define PGM_RAM_RANGE_FLAGS_AD_HOC_ROM RT_BIT(21)
1178	/** Ad hoc RAM range for an MMIO mapping. */
1179	#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO RT_BIT(22)
1180	/** Ad hoc RAM range for an MMIO2 mapping. */
1181	#define PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2 RT_BIT(23)
1182	/** @} */
1183
1184	/** Tests if a RAM range is an ad hoc one or not.
1185	* @returns true/false.
1186	* @param pRam The RAM range.
1187	*/
1188	#define PGM_RAM_RANGE_IS_AD_HOC(pRam) \
1189	(!!( (pRam)->fFlags & (PGM_RAM_RANGE_FLAGS_AD_HOC_ROM \| PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO \| PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO2) ) )
1190
1191
1192	/**
1193	* Per page tracking structure for ROM image.
1194	*
1195	* A ROM image may have a shadow page, in which case we may have two pages
1196	* backing it. This structure contains the PGMPAGE for both while
1197	* PGMRAMRANGE have a copy of the active one. It is important that these
1198	* aren't out of sync in any regard other than page pool tracking data.
1199	*/
1200	typedef struct PGMROMPAGE
1201	{
1202	/** The page structure for the virgin ROM page. */
1203	PGMPAGE Virgin;
1204	/** The page structure for the shadow RAM page. */
1205	PGMPAGE Shadow;
1206	/** The current protection setting. */
1207	PGMROMPROT enmProt;
1208	/** Live save status information. Makes use of unused alignment space. */
1209	struct
1210	{
1211	/** The previous protection value. */
1212	uint8_t u8Prot;
1213	/** Written to flag set by the handler. */
1214	bool fWrittenTo;
1215	/** Whether the shadow page is dirty or not. */
1216	bool fDirty;
1217	/** Whether it was dirtied in the recently. */
1218	bool fDirtiedRecently;
1219	} LiveSave;
1220	} PGMROMPAGE;
1221	AssertCompileSizeAlignment(PGMROMPAGE, 8);
1222	/** Pointer to a ROM page tracking structure. */
1223	typedef PGMROMPAGE *PPGMROMPAGE;
1224
1225
1226	/**
1227	* A registered ROM image.
1228	*
1229	* This is needed to keep track of ROM image since they generally intrude
1230	* into a PGMRAMRANGE. It also keeps track of additional info like the
1231	* two page sets (read-only virgin and read-write shadow), the current
1232	* state of each page.
1233	*
1234	* Because access handlers cannot easily be executed in a different
1235	* context, the ROM ranges needs to be accessible and in all contexts.
1236	*/
1237	typedef struct PGMROMRANGE
1238	{
1239	/** Pointer to the next range - R3. */
1240	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1241	/** Pointer to the next range - R0. */
1242	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1243	/** Pointer to the next range - RC. */
1244	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1245	/** Pointer alignment */
1246	RTRCPTR RCPtrAlignment;
1247	/** Address of the range. */
1248	RTGCPHYS GCPhys;
1249	/** Address of the last byte in the range. */
1250	RTGCPHYS GCPhysLast;
1251	/** Size of the range. */
1252	RTGCPHYS cb;
1253	/** The flags (PGMPHYS_ROM_FLAGS_). /
1254	uint32_t fFlags;
1255	/** The saved state range ID. */
1256	uint8_t idSavedState;
1257	/** Alignment padding. */
1258	uint8_t au8Alignment[3];
1259	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1260	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 6 : 2];
1261	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1262	* This is used for strictness checks. */
1263	R3PTRTYPE(const void *) pvOriginal;
1264	/** The ROM description. */
1265	R3PTRTYPE(const char *) pszDesc;
1266	/** The per page tracking structures. */
1267	PGMROMPAGE aPages[1];
1268	} PGMROMRANGE;
1269	/** Pointer to a ROM range. */
1270	typedef PGMROMRANGE *PPGMROMRANGE;
1271
1272
1273	/**
1274	* Live save per page data for an MMIO2 page.
1275	*
1276	* Not using PGMLIVESAVERAMPAGE here because we cannot use normal write monitoring
1277	* of MMIO2 pages. The current approach is using some optimisitic SHA-1 +
1278	* CRC-32 for detecting changes as well as special handling of zero pages. This
1279	* is a TEMPORARY measure which isn't perfect, but hopefully it is good enough
1280	* for speeding things up. (We're using SHA-1 and not SHA-256 or SHA-512
1281	* because of speed (2.5x and 6x slower).)
1282	*
1283	* @todo Implement dirty MMIO2 page reporting that can be enabled during live
1284	* save but normally is disabled. Since we can write monitore guest
1285	* accesses on our own, we only need this for host accesses. Shouldn't be
1286	* too difficult for DevVGA, VMMDev might be doable, the planned
1287	* networking fun will be fun since it involves ring-0.
1288	*/
1289	typedef struct PGMLIVESAVEMMIO2PAGE
1290	{
1291	/** Set if the page is considered dirty. */
1292	bool fDirty;
1293	/** The number of scans this page has remained unchanged for.
1294	* Only updated for dirty pages. */
1295	uint8_t cUnchangedScans;
1296	/** Whether this page was zero at the last scan. */
1297	bool fZero;
1298	/** Alignment padding. */
1299	bool fReserved;
1300	/** CRC-32 for the first half of the page.
1301	* This is used together with u32CrcH2 to quickly detect changes in the page
1302	* during the non-final passes. */
1303	uint32_t u32CrcH1;
1304	/** CRC-32 for the second half of the page. */
1305	uint32_t u32CrcH2;
1306	/** SHA-1 for the saved page.
1307	* This is used in the final pass to skip pages without changes. */
1308	uint8_t abSha1Saved[RTSHA1_HASH_SIZE];
1309	} PGMLIVESAVEMMIO2PAGE;
1310	/** Pointer to a live save status data for an MMIO2 page. */
1311	typedef PGMLIVESAVEMMIO2PAGE *PPGMLIVESAVEMMIO2PAGE;
1312
1313	/**
1314	* A registered MMIO2 (= Device RAM) range.
1315	*
1316	* There are a few reason why we need to keep track of these
1317	* registrations. One of them is the deregistration & cleanup stuff,
1318	* while another is that the PGMRAMRANGE associated with such a region may
1319	* have to be removed from the ram range list.
1320	*
1321	* Overlapping with a RAM range has to be 100% or none at all. The pages
1322	* in the existing RAM range must not be ROM nor MMIO. A guru meditation
1323	* will be raised if a partial overlap or an overlap of ROM pages is
1324	* encountered. On an overlap we will free all the existing RAM pages and
1325	* put in the ram range pages instead.
1326	*/
1327	typedef struct PGMMMIO2RANGE
1328	{
1329	/** The owner of the range. (a device) */
1330	PPDMDEVINSR3 pDevInsR3;
1331	/** Pointer to the ring-3 mapping of the allocation. */
1332	RTR3PTR pvR3;
1333	/** Pointer to the next range - R3. */
1334	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1335	/** Whether it's mapped or not. */
1336	bool fMapped;
1337	/** Whether it's overlapping or not. */
1338	bool fOverlapping;
1339	/** The PCI region number.
1340	* @remarks This ASSUMES that nobody will ever really need to have multiple
1341	* PCI devices with matching MMIO region numbers on a single device. */
1342	uint8_t iRegion;
1343	/** The saved state range ID. */
1344	uint8_t idSavedState;
1345	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundrary. */
1346	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 12 : 12];
1347	/** Live save per page tracking data. */
1348	R3PTRTYPE(PPGMLIVESAVEMMIO2PAGE) paLSPages;
1349	/** The associated RAM range. */
1350	PGMRAMRANGE RamRange;
1351	} PGMMMIO2RANGE;
1352	/** Pointer to a MMIO2 range. */
1353	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1354
1355
1356
1357
1358	/**
1359	* PGMPhysRead/Write cache entry
1360	*/
1361	typedef struct PGMPHYSCACHEENTRY
1362	{
1363	/** R3 pointer to physical page. */
1364	R3PTRTYPE(uint8_t *) pbR3;
1365	/** GC Physical address for cache entry */
1366	RTGCPHYS GCPhys;
1367	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1368	RTGCPHYS u32Padding0; /*< alignment padding. /
1369	#endif
1370	} PGMPHYSCACHEENTRY;
1371
1372	/**
1373	* PGMPhysRead/Write cache to reduce REM memory access overhead
1374	*/
1375	typedef struct PGMPHYSCACHE
1376	{
1377	/** Bitmap of valid cache entries */
1378	uint64_t aEntries;
1379	/** Cache entries */
1380	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1381	} PGMPHYSCACHE;
1382
1383
1384	/** Pointer to an allocation chunk ring-3 mapping. */
1385	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1386	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1387	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1388
1389	/**
1390	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1391	*
1392	* The primary tree (Core) uses the chunk id as key.
1393	* The secondary tree (AgeCore) is used for ageing and uses ageing sequence number as key.
1394	*/
1395	typedef struct PGMCHUNKR3MAP
1396	{
1397	/** The key is the chunk id. */
1398	AVLU32NODECORE Core;
1399	/** The key is the ageing sequence number. */
1400	AVLLU32NODECORE AgeCore;
1401	/** The current age thingy. */
1402	uint32_t iAge;
1403	/** The current reference count. */
1404	uint32_t volatile cRefs;
1405	/** The current permanent reference count. */
1406	uint32_t volatile cPermRefs;
1407	/** The mapping address. */
1408	void *pv;
1409	} PGMCHUNKR3MAP;
1410
1411	/**
1412	* Allocation chunk ring-3 mapping TLB entry.
1413	*/
1414	typedef struct PGMCHUNKR3MAPTLBE
1415	{
1416	/** The chunk id. */
1417	uint32_t volatile idChunk;
1418	#if HC_ARCH_BITS == 64
1419	uint32_t u32Padding; /*< alignment padding. /
1420	#endif
1421	/** The chunk map. */
1422	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1423	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1424	#else
1425	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1426	#endif
1427	} PGMCHUNKR3MAPTLBE;
1428	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1429	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1430
1431	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1432	* @remark Must be a power of two value. */
1433	#define PGM_CHUNKR3MAPTLB_ENTRIES 64
1434
1435	/**
1436	* Allocation chunk ring-3 mapping TLB.
1437	*
1438	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1439	* At first glance this might look kinda odd since AVL trees are
1440	* supposed to give the most optimial lookup times of all trees
1441	* due to their balancing. However, take a tree with 1023 nodes
1442	* in it, that's 10 levels, meaning that most searches has to go
1443	* down 9 levels before they find what they want. This isn't fast
1444	* compared to a TLB hit. There is the factor of cache misses,
1445	* and of course the problem with trees and branch prediction.
1446	* This is why we use TLBs in front of most of the trees.
1447	*
1448	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1449	* difficult when we switch to the new inlined AVL trees (from kStuff).
1450	*/
1451	typedef struct PGMCHUNKR3MAPTLB
1452	{
1453	/** The TLB entries. */
1454	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1455	} PGMCHUNKR3MAPTLB;
1456
1457	/**
1458	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1459	* @returns Chunk TLB index.
1460	* @param idChunk The Chunk ID.
1461	*/
1462	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1463
1464
1465	/**
1466	* Ring-3 guest page mapping TLB entry.
1467	* @remarks used in ring-0 as well at the moment.
1468	*/
1469	typedef struct PGMPAGER3MAPTLBE
1470	{
1471	/** Address of the page. */
1472	RTGCPHYS volatile GCPhys;
1473	/** The guest page. */
1474	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1475	R3PTRTYPE(PPGMPAGE) volatile pPage;
1476	#else
1477	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1478	#endif
1479	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1480	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1481	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1482	#else
1483	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1484	#endif
1485	/** The address */
1486	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1487	R3PTRTYPE(void *) volatile pv;
1488	#else
1489	R3R0PTRTYPE(void *) volatile pv;
1490	#endif
1491	#if HC_ARCH_BITS == 32
1492	uint32_t u32Padding; /*< alignment padding. /
1493	#endif
1494	} PGMPAGER3MAPTLBE;
1495	/** Pointer to an entry in the HC physical TLB. */
1496	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1497
1498
1499	/** The number of entries in the ring-3 guest page mapping TLB.
1500	* @remarks The value must be a power of two. */
1501	#define PGM_PAGER3MAPTLB_ENTRIES 256
1502
1503	/**
1504	* Ring-3 guest page mapping TLB.
1505	* @remarks used in ring-0 as well at the moment.
1506	*/
1507	typedef struct PGMPAGER3MAPTLB
1508	{
1509	/** The TLB entries. */
1510	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1511	} PGMPAGER3MAPTLB;
1512	/** Pointer to the ring-3 guest page mapping TLB. */
1513	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1514
1515	/**
1516	* Calculates the index of the TLB entry for the specified guest page.
1517	* @returns Physical TLB index.
1518	* @param GCPhys The guest physical address.
1519	*/
1520	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1521
1522
1523	/**
1524	* Mapping cache usage set entry.
1525	*
1526	* @remarks 16-bit ints was choosen as the set is not expected to be used beyond
1527	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1528	* cache. If it's extended to include ring-3, well, then something will
1529	* have be changed here...
1530	*/
1531	typedef struct PGMMAPSETENTRY
1532	{
1533	/** The mapping cache index. */
1534	uint16_t iPage;
1535	/** The number of references.
1536	* The max is UINT16_MAX - 1. */
1537	uint16_t cRefs;
1538	#if HC_ARCH_BITS == 64
1539	uint32_t alignment;
1540	#endif
1541	/** Pointer to the page. */
1542	RTR0PTR pvPage;
1543	/** The physical address for this entry. */
1544	RTHCPHYS HCPhys;
1545	} PGMMAPSETENTRY;
1546	/** Pointer to a mapping cache usage set entry. */
1547	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1548
1549	/**
1550	* Mapping cache usage set.
1551	*
1552	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1553	* done during exits / traps. The set is
1554	*/
1555	typedef struct PGMMAPSET
1556	{
1557	/** The number of occupied entries.
1558	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1559	* dynamic mappings. */
1560	uint32_t cEntries;
1561	/** The start of the current subset.
1562	* This is UINT32_MAX if no subset is currently open. */
1563	uint32_t iSubset;
1564	/** The index of the current CPU, only valid if the set is open. */
1565	int32_t iCpu;
1566	uint32_t alignment;
1567	/** The entries. */
1568	PGMMAPSETENTRY aEntries[64];
1569	/** HCPhys -> iEntry fast lookup table.
1570	* Use PGMMAPSET_HASH for hashing.
1571	* The entries may or may not be valid, check against cEntries. */
1572	uint8_t aiHashTable[128];
1573	} PGMMAPSET;
1574	AssertCompileSizeAlignment(PGMMAPSET, 8);
1575	/** Pointer to the mapping cache set. */
1576	typedef PGMMAPSET *PPGMMAPSET;
1577
1578	/** PGMMAPSET::cEntries value for a closed set. */
1579	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1580
1581	/** Hash function for aiHashTable. */
1582	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1583
1584	/** The max fill size (strict builds). */
1585	#define PGMMAPSET_MAX_FILL (64U * 80U / 100U)
1586
1587
1588	/** @name Context neutrual page mapper TLB.
1589	*
1590	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1591	* code is writting in a kind of context neutrual way. Time will show whether
1592	* this actually makes sense or not...
1593	*
1594	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1595	* context ends up using a global mapping cache on some platforms
1596	* (darwin).
1597	*
1598	* @{ */
1599	/** @typedef PPGMPAGEMAPTLB
1600	* The page mapper TLB pointer type for the current context. */
1601	/** @typedef PPGMPAGEMAPTLB
1602	* The page mapper TLB entry pointer type for the current context. */
1603	/** @typedef PPGMPAGEMAPTLB
1604	* The page mapper TLB entry pointer pointer type for the current context. */
1605	/** @def PGM_PAGEMAPTLB_ENTRIES
1606	* The number of TLB entries in the page mapper TLB for the current context. */
1607	/** @def PGM_PAGEMAPTLB_IDX
1608	* Calculate the TLB index for a guest physical address.
1609	* @returns The TLB index.
1610	* @param GCPhys The guest physical address. */
1611	/** @typedef PPGMPAGEMAP
1612	* Pointer to a page mapper unit for current context. */
1613	/** @typedef PPPGMPAGEMAP
1614	* Pointer to a page mapper unit pointer for current context. */
1615	#ifdef IN_RC
1616	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1617	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1618	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1619	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1620	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1621	typedef void * PPGMPAGEMAP;
1622	typedef void ** PPPGMPAGEMAP;
1623	//#elif IN_RING0
1624	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1625	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1626	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1627	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1628	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1629	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1630	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1631	#else
1632	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1633	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1634	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1635	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1636	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1637	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1638	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1639	#endif
1640	/** @} */
1641
1642
1643	/** @name PGM Pool Indexes.
1644	* Aka. the unique shadow page identifier.
1645	* @{ */
1646	/** NIL page pool IDX. */
1647	#define NIL_PGMPOOL_IDX 0
1648	/** The first normal index. */
1649	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1650	/** Page directory (32-bit root). */
1651	#define PGMPOOL_IDX_PD 1
1652	/** Page Directory Pointer Table (PAE root). */
1653	#define PGMPOOL_IDX_PDPT 2
1654	/** AMD64 CR3 level index.*/
1655	#define PGMPOOL_IDX_AMD64_CR3 3
1656	/** Nested paging root.*/
1657	#define PGMPOOL_IDX_NESTED_ROOT 4
1658	/** The first normal index. */
1659	#define PGMPOOL_IDX_FIRST 5
1660	/** The last valid index. (inclusive, 14 bits) */
1661	#define PGMPOOL_IDX_LAST 0x3fff
1662	/** @} */
1663
1664	/** The NIL index for the parent chain. */
1665	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1666	#define NIL_PGMPOOL_PRESENT_INDEX ((uint16_t)0xffff)
1667
1668	/**
1669	* Node in the chain linking a shadowed page to it's parent (user).
1670	*/
1671	#pragma pack(1)
1672	typedef struct PGMPOOLUSER
1673	{
1674	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1675	uint16_t iNext;
1676	/** The user page index. */
1677	uint16_t iUser;
1678	/** Index into the user table. */
1679	uint32_t iUserTable;
1680	} PGMPOOLUSER, *PPGMPOOLUSER;
1681	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1682	#pragma pack()
1683
1684
1685	/** The NIL index for the phys ext chain. */
1686	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1687
1688	/**
1689	* Node in the chain of physical cross reference extents.
1690	* @todo Calling this an 'extent' is not quite right, find a better name.
1691	*/
1692	#pragma pack(1)
1693	typedef struct PGMPOOLPHYSEXT
1694	{
1695	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1696	uint16_t iNext;
1697	/** The user page index. */
1698	uint16_t aidx[3];
1699	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1700	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1701	#pragma pack()
1702
1703
1704	/**
1705	* The kind of page that's being shadowed.
1706	*/
1707	typedef enum PGMPOOLKIND
1708	{
1709	/** The virtual invalid 0 entry. */
1710	PGMPOOLKIND_INVALID = 0,
1711	/** The entry is free (=unused). */
1712	PGMPOOLKIND_FREE,
1713
1714	/** Shw: 32-bit page table; Gst: no paging */
1715	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1716	/** Shw: 32-bit page table; Gst: 32-bit page table. */
1717	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1718	/** Shw: 32-bit page table; Gst: 4MB page. */
1719	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1720	/** Shw: PAE page table; Gst: no paging */
1721	PGMPOOLKIND_PAE_PT_FOR_PHYS,
1722	/** Shw: PAE page table; Gst: 32-bit page table. */
1723	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1724	/** Shw: PAE page table; Gst: Half of a 4MB page. */
1725	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1726	/** Shw: PAE page table; Gst: PAE page table. */
1727	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1728	/** Shw: PAE page table; Gst: 2MB page. */
1729	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1730
1731	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1732	PGMPOOLKIND_32BIT_PD,
1733	/** Shw: 32-bit page directory. Gst: no paging. */
1734	PGMPOOLKIND_32BIT_PD_PHYS,
1735	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1736	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1737	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1738	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1739	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1740	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1741	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1742	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1743	/** Shw: PAE page directory; Gst: PAE page directory. */
1744	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1745	/** Shw: PAE page directory; Gst: no paging. */
1746	PGMPOOLKIND_PAE_PD_PHYS,
1747
1748	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1749	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1750	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1751	PGMPOOLKIND_PAE_PDPT,
1752	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1753	PGMPOOLKIND_PAE_PDPT_PHYS,
1754
1755	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1756	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1757	/** Shw: 64-bit page directory pointer table; Gst: no paging */
1758	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1759	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1760	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1761	/** Shw: 64-bit page directory table; Gst: no paging */
1762	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
1763
1764	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1765	PGMPOOLKIND_64BIT_PML4,
1766
1767	/** Shw: EPT page directory pointer table; Gst: no paging */
1768	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1769	/** Shw: EPT page directory table; Gst: no paging */
1770	PGMPOOLKIND_EPT_PD_FOR_PHYS,
1771	/** Shw: EPT page table; Gst: no paging */
1772	PGMPOOLKIND_EPT_PT_FOR_PHYS,
1773
1774	/** Shw: Root Nested paging table. */
1775	PGMPOOLKIND_ROOT_NESTED,
1776
1777	/** The last valid entry. */
1778	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1779	} PGMPOOLKIND;
1780
1781	/**
1782	* The access attributes of the page; only applies to big pages.
1783	*/
1784	typedef enum
1785	{
1786	PGMPOOLACCESS_DONTCARE = 0,
1787	PGMPOOLACCESS_USER_RW,
1788	PGMPOOLACCESS_USER_R,
1789	PGMPOOLACCESS_USER_RW_NX,
1790	PGMPOOLACCESS_USER_R_NX,
1791	PGMPOOLACCESS_SUPERVISOR_RW,
1792	PGMPOOLACCESS_SUPERVISOR_R,
1793	PGMPOOLACCESS_SUPERVISOR_RW_NX,
1794	PGMPOOLACCESS_SUPERVISOR_R_NX
1795	} PGMPOOLACCESS;
1796
1797	/**
1798	* The tracking data for a page in the pool.
1799	*/
1800	typedef struct PGMPOOLPAGE
1801	{
1802	/** AVL node code with the (R3) physical address of this page. */
1803	AVLOHCPHYSNODECORE Core;
1804	/** Pointer to the R3 mapping of the page. */
1805	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1806	R3PTRTYPE(void *) pvPageR3;
1807	#else
1808	R3R0PTRTYPE(void *) pvPageR3;
1809	#endif
1810	/** The guest physical address. */
1811	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
1812	uint32_t Alignment0;
1813	#endif
1814	RTGCPHYS GCPhys;
1815
1816	/** Access handler statistics to determine whether the guest is (re)initializing a page table. */
1817	RTGCPTR pvLastAccessHandlerRip;
1818	RTGCPTR pvLastAccessHandlerFault;
1819	uint64_t cLastAccessHandlerCount;
1820
1821	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1822	uint8_t enmKind;
1823	/** The subkind of page we're shadowing. (This is really a PGMPOOLACCESS enum.) */
1824	uint8_t enmAccess;
1825	/** The index of this page. */
1826	uint16_t idx;
1827	/** The next entry in the list this page currently resides in.
1828	* It's either in the free list or in the GCPhys hash. */
1829	uint16_t iNext;
1830	#ifdef PGMPOOL_WITH_USER_TRACKING
1831	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1832	uint16_t iUserHead;
1833	/** The number of present entries. */
1834	uint16_t cPresent;
1835	/** The first entry in the table which is present. */
1836	uint16_t iFirstPresent;
1837	#endif
1838	#ifdef PGMPOOL_WITH_MONITORING
1839	/** The number of modifications to the monitored page. */
1840	uint16_t cModifications;
1841	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
1842	uint16_t iModifiedNext;
1843	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
1844	uint16_t iModifiedPrev;
1845	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1846	uint16_t iMonitoredNext;
1847	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1848	uint16_t iMonitoredPrev;
1849	#endif
1850	#ifdef PGMPOOL_WITH_CACHE
1851	/** The next page in the age list. */
1852	uint16_t iAgeNext;
1853	/** The previous page in the age list. */
1854	uint16_t iAgePrev;
1855	#endif /* PGMPOOL_WITH_CACHE */
1856	/** Used to indicate that the page is zeroed. */
1857	bool fZeroed;
1858	/** Used to indicate that a PT has non-global entries. */
1859	bool fSeenNonGlobal;
1860	/** Used to indicate that we're monitoring writes to the guest page. */
1861	bool fMonitored;
1862	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1863	* (All pages are in the age list.) */
1864	bool fCached;
1865	/** This is used by the R3 access handlers when invoked by an async thread.
1866	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1867	bool volatile fReusedFlushPending;
1868	/** Used to mark the page as dirty (write monitoring if temporarily off. */
1869	bool fDirty;
1870
1871	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
1872	uint32_t cLocked;
1873	uint32_t idxDirty;
1874	RTGCPTR pvDirtyFault;
1875	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
1876	/** Pointer to a const pool page. */
1877	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1878
1879
1880	#ifdef PGMPOOL_WITH_CACHE
1881	/** The hash table size. */
1882	# define PGMPOOL_HASH_SIZE 0x40
1883	/** The hash function. */
1884	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1885	#endif
1886
1887
1888	/**
1889	* The shadow page pool instance data.
1890	*
1891	* It's all one big allocation made at init time, except for the
1892	* pages that is. The user nodes follows immediatly after the
1893	* page structures.
1894	*/
1895	typedef struct PGMPOOL
1896	{
1897	/** The VM handle - R3 Ptr. */
1898	PVMR3 pVMR3;
1899	/** The VM handle - R0 Ptr. */
1900	PVMR0 pVMR0;
1901	/** The VM handle - RC Ptr. */
1902	PVMRC pVMRC;
1903	/** The max pool size. This includes the special IDs. */
1904	uint16_t cMaxPages;
1905	/** The current pool size. */
1906	uint16_t cCurPages;
1907	/** The head of the free page list. */
1908	uint16_t iFreeHead;
1909	/* Padding. */
1910	uint16_t u16Padding;
1911	#ifdef PGMPOOL_WITH_USER_TRACKING
1912	/** Head of the chain of free user nodes. */
1913	uint16_t iUserFreeHead;
1914	/** The number of user nodes we've allocated. */
1915	uint16_t cMaxUsers;
1916	/** The number of present page table entries in the entire pool. */
1917	uint32_t cPresent;
1918	/** Pointer to the array of user nodes - RC pointer. */
1919	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
1920	/** Pointer to the array of user nodes - R3 pointer. */
1921	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
1922	/** Pointer to the array of user nodes - R0 pointer. */
1923	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
1924	#endif /* PGMPOOL_WITH_USER_TRACKING */
1925	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1926	/** Head of the chain of free phys ext nodes. */
1927	uint16_t iPhysExtFreeHead;
1928	/** The number of user nodes we've allocated. */
1929	uint16_t cMaxPhysExts;
1930	/** Pointer to the array of physical xref extent - RC pointer. */
1931	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
1932	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
1933	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
1934	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
1935	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
1936	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1937	#ifdef PGMPOOL_WITH_CACHE
1938	/** Hash table for GCPhys addresses. */
1939	uint16_t aiHash[PGMPOOL_HASH_SIZE];
1940	/** The head of the age list. */
1941	uint16_t iAgeHead;
1942	/** The tail of the age list. */
1943	uint16_t iAgeTail;
1944	/** Set if the cache is enabled. */
1945	bool fCacheEnabled;
1946	/** Alignment padding. */
1947	bool afPadding1[3];
1948	#endif /* PGMPOOL_WITH_CACHE */
1949	#ifdef PGMPOOL_WITH_MONITORING
1950	/** Head of the list of modified pages. */
1951	uint16_t iModifiedHead;
1952	/** The current number of modified pages. */
1953	uint16_t cModifiedPages;
1954	/** Access handler, RC. */
1955	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
1956	/** Access handler, R0. */
1957	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
1958	/** Access handler, R3. */
1959	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
1960	/** The access handler description (R3 ptr). */
1961	R3PTRTYPE(const char *) pszAccessHandler;
1962	# if HC_ARCH_BITS == 32
1963	/** Alignment padding. */
1964	uint32_t u32Padding2;
1965	# endif
1966	/* Next available slot. */
1967	uint32_t idxFreeDirtyPage;
1968	/* Number of active dirty pages. */
1969	uint32_t cDirtyPages;
1970	/* Array of current dirty pgm pool page indices. */
1971	uint16_t aIdxDirtyPages[16];
1972	uint64_t aDirtyPages[16][512];
1973	#endif /* PGMPOOL_WITH_MONITORING */
1974	/** The number of pages currently in use. */
1975	uint16_t cUsedPages;
1976	#ifdef VBOX_WITH_STATISTICS
1977	/** The high water mark for cUsedPages. */
1978	uint16_t cUsedPagesHigh;
1979	uint32_t Alignment1; /*< Align the next member on a 64-bit boundrary. /
1980	/** Profiling pgmPoolAlloc(). */
1981	STAMPROFILEADV StatAlloc;
1982	/** Profiling pgmR3PoolClearDoIt(). */
1983	STAMPROFILE StatClearAll;
1984	/** Profiling pgmR3PoolReset(). */
1985	STAMPROFILE StatR3Reset;
1986	/** Profiling pgmPoolFlushPage(). */
1987	STAMPROFILE StatFlushPage;
1988	/** Profiling pgmPoolFree(). */
1989	STAMPROFILE StatFree;
1990	/** Counting explicit flushes by PGMPoolFlushPage(). */
1991	STAMCOUNTER StatForceFlushPage;
1992	/** Counting explicit flushes of dirty pages by PGMPoolFlushPage(). */
1993	STAMCOUNTER StatForceFlushDirtyPage;
1994	/** Counting flushes for reused pages. */
1995	STAMCOUNTER StatForceFlushReused;
1996	/** Profiling time spent zeroing pages. */
1997	STAMPROFILE StatZeroPage;
1998	# ifdef PGMPOOL_WITH_USER_TRACKING
1999	/** Profiling of pgmPoolTrackDeref. */
2000	STAMPROFILE StatTrackDeref;
2001	/** Profiling pgmTrackFlushGCPhysPT. */
2002	STAMPROFILE StatTrackFlushGCPhysPT;
2003	/** Profiling pgmTrackFlushGCPhysPTs. */
2004	STAMPROFILE StatTrackFlushGCPhysPTs;
2005	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
2006	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
2007	/** Number of times we've been out of user records. */
2008	STAMCOUNTER StatTrackFreeUpOneUser;
2009	/** Nr of flushed entries. */
2010	STAMCOUNTER StatTrackFlushEntry;
2011	/** Nr of updated entries. */
2012	STAMCOUNTER StatTrackFlushEntryKeep;
2013	# endif
2014	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2015	/** Profiling deref activity related tracking GC physical pages. */
2016	STAMPROFILE StatTrackDerefGCPhys;
2017	/** Number of linear searches for a HCPhys in the ram ranges. */
2018	STAMCOUNTER StatTrackLinearRamSearches;
2019	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
2020	STAMCOUNTER StamTrackPhysExtAllocFailures;
2021	# endif
2022	# ifdef PGMPOOL_WITH_MONITORING
2023	/** Profiling the RC/R0 access handler. */
2024	STAMPROFILE StatMonitorRZ;
2025	/** Times we've failed interpreting the instruction. */
2026	STAMCOUNTER StatMonitorRZEmulateInstr;
2027	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
2028	STAMPROFILE StatMonitorRZFlushPage;
2029	/* Times we've detected a page table reinit. */
2030	STAMCOUNTER StatMonitorRZFlushReinit;
2031	/** Counting flushes for pages that are modified too often. */
2032	STAMCOUNTER StatMonitorRZFlushModOverflow;
2033	/** Times we've detected fork(). */
2034	STAMCOUNTER StatMonitorRZFork;
2035	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
2036	STAMPROFILE StatMonitorRZHandled;
2037	/** Times we've failed interpreting a patch code instruction. */
2038	STAMCOUNTER StatMonitorRZIntrFailPatch1;
2039	/** Times we've failed interpreting a patch code instruction during flushing. */
2040	STAMCOUNTER StatMonitorRZIntrFailPatch2;
2041	/** The number of times we've seen rep prefixes we can't handle. */
2042	STAMCOUNTER StatMonitorRZRepPrefix;
2043	/** Profiling the REP STOSD cases we've handled. */
2044	STAMPROFILE StatMonitorRZRepStosd;
2045	/** Nr of handled PT faults. */
2046	STAMCOUNTER StatMonitorRZFaultPT;
2047	/** Nr of handled PD faults. */
2048	STAMCOUNTER StatMonitorRZFaultPD;
2049	/** Nr of handled PDPT faults. */
2050	STAMCOUNTER StatMonitorRZFaultPDPT;
2051	/** Nr of handled PML4 faults. */
2052	STAMCOUNTER StatMonitorRZFaultPML4;
2053
2054	/** Profiling the R3 access handler. */
2055	STAMPROFILE StatMonitorR3;
2056	/** Times we've failed interpreting the instruction. */
2057	STAMCOUNTER StatMonitorR3EmulateInstr;
2058	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
2059	STAMPROFILE StatMonitorR3FlushPage;
2060	/* Times we've detected a page table reinit. */
2061	STAMCOUNTER StatMonitorR3FlushReinit;
2062	/** Counting flushes for pages that are modified too often. */
2063	STAMCOUNTER StatMonitorR3FlushModOverflow;
2064	/** Times we've detected fork(). */
2065	STAMCOUNTER StatMonitorR3Fork;
2066	/** Profiling the R3 access we've handled (except REP STOSD). */
2067	STAMPROFILE StatMonitorR3Handled;
2068	/** The number of times we've seen rep prefixes we can't handle. */
2069	STAMCOUNTER StatMonitorR3RepPrefix;
2070	/** Profiling the REP STOSD cases we've handled. */
2071	STAMPROFILE StatMonitorR3RepStosd;
2072	/** Nr of handled PT faults. */
2073	STAMCOUNTER StatMonitorR3FaultPT;
2074	/** Nr of handled PD faults. */
2075	STAMCOUNTER StatMonitorR3FaultPD;
2076	/** Nr of handled PDPT faults. */
2077	STAMCOUNTER StatMonitorR3FaultPDPT;
2078	/** Nr of handled PML4 faults. */
2079	STAMCOUNTER StatMonitorR3FaultPML4;
2080	/** The number of times we're called in an async thread an need to flush. */
2081	STAMCOUNTER StatMonitorR3Async;
2082	/** Times we've called pgmPoolResetDirtyPages (and there were dirty page). */
2083	STAMCOUNTER StatResetDirtyPages;
2084	/** Times we've called pgmPoolAddDirtyPage. */
2085	STAMCOUNTER StatDirtyPage;
2086	/** Times we've had to flush duplicates for dirty page management. */
2087	STAMCOUNTER StatDirtyPageDupFlush;
2088	/** Times we've had to flush because of overflow. */
2089	STAMCOUNTER StatDirtyPageOverFlowFlush;
2090
2091	/** The high wather mark for cModifiedPages. */
2092	uint16_t cModifiedPagesHigh;
2093	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundrary. /
2094	# endif
2095	# ifdef PGMPOOL_WITH_CACHE
2096	/** The number of cache hits. */
2097	STAMCOUNTER StatCacheHits;
2098	/** The number of cache misses. */
2099	STAMCOUNTER StatCacheMisses;
2100	/** The number of times we've got a conflict of 'kind' in the cache. */
2101	STAMCOUNTER StatCacheKindMismatches;
2102	/** Number of times we've been out of pages. */
2103	STAMCOUNTER StatCacheFreeUpOne;
2104	/** The number of cacheable allocations. */
2105	STAMCOUNTER StatCacheCacheable;
2106	/** The number of uncacheable allocations. */
2107	STAMCOUNTER StatCacheUncacheable;
2108	# endif
2109	#else
2110	uint32_t Alignment3; /*< Align the next member on a 64-bit boundrary. /
2111	#endif
2112	/** The AVL tree for looking up a page by its HC physical address. */
2113	AVLOHCPHYSTREE HCPhysTree;
2114	uint32_t Alignment4; /*< Align the next member on a 64-bit boundrary. /
2115	/** Array of pages. (cMaxPages in length)
2116	* The Id is the index into thist array.
2117	*/
2118	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
2119	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
2120	#ifdef PGMPOOL_WITH_MONITORING
2121	AssertCompileMemberAlignment(PGMPOOL, iModifiedHead, 8);
2122	AssertCompileMemberAlignment(PGMPOOL, aDirtyPages, 8);
2123	#endif
2124	AssertCompileMemberAlignment(PGMPOOL, cUsedPages, 8);
2125	#ifdef VBOX_WITH_STATISTICS
2126	AssertCompileMemberAlignment(PGMPOOL, StatAlloc, 8);
2127	#endif
2128	AssertCompileMemberAlignment(PGMPOOL, aPages, 8);
2129
2130
2131	/** @def PGMPOOL_PAGE_2_PTR
2132	* Maps a pool page pool into the current context.
2133	*
2134	* @returns VBox status code.
2135	* @param pVM The VM handle.
2136	* @param pPage The pool page.
2137	*
2138	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2139	* small page window employeed by that function. Be careful.
2140	* @remark There is no need to assert on the result.
2141	*/
2142	#if defined(IN_RC)
2143	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
2144	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2145	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
2146	#elif defined(VBOX_STRICT)
2147	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageStrict(pPage)
2148	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
2149	{
2150	Assert(pPage && pPage->pvPageR3);
2151	return pPage->pvPageR3;
2152	}
2153	#else
2154	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) ((pPage)->pvPageR3)
2155	#endif
2156
2157	/** @def PGMPOOL_PAGE_2_PTR_BY_PGM
2158	* Maps a pool page pool into the current context.
2159	*
2160	* @returns VBox status code.
2161	* @param pPGM Pointer to the PGM instance data.
2162	* @param pPage The pool page.
2163	*
2164	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2165	* small page window employeed by that function. Be careful.
2166	* @remark There is no need to assert on the result.
2167	*/
2168	#if defined(IN_RC)
2169	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
2170	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2171	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
2172	#else
2173	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGM2VM(pPGM), pPage)
2174	#endif
2175
2176	/** @def PGMPOOL_PAGE_2_PTR_BY_PGMCPU
2177	* Maps a pool page pool into the current context.
2178	*
2179	* @returns VBox status code.
2180	* @param pPGM Pointer to the PGMCPU instance data.
2181	* @param pPage The pool page.
2182	*
2183	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
2184	* small page window employeed by that function. Be careful.
2185	* @remark There is no need to assert on the result.
2186	*/
2187	#if defined(IN_RC)
2188	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
2189	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2190	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
2191	#else
2192	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGMCPU2VM(pPGM), pPage)
2193	#endif
2194
2195
2196	/** @name Per guest page tracking data.
2197	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
2198	* is to use more bits for it and split it up later on. But for now we'll play
2199	* safe and change as little as possible.
2200	*
2201	* The 16-bit word has two parts:
2202	*
2203	* The first 14-bit forms the @a idx field. It is either the index of a page in
2204	* the shadow page pool, or and index into the extent list.
2205	*
2206	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
2207	* shadow page pool references to the page. If cRefs equals
2208	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
2209	* (misnomer) table and not the shadow page pool.
2210	*
2211	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
2212	* the 16-bit word.
2213	*
2214	* @{ */
2215	/** The shift count for getting to the cRefs part. */
2216	#define PGMPOOL_TD_CREFS_SHIFT 14
2217	/** The mask applied after shifting the tracking data down by
2218	* PGMPOOL_TD_CREFS_SHIFT. */
2219	#define PGMPOOL_TD_CREFS_MASK 0x3
2220	/** The cRef value used to indiciate that the idx is the head of a
2221	* physical cross reference list. */
2222	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
2223	/** The shift used to get idx. */
2224	#define PGMPOOL_TD_IDX_SHIFT 0
2225	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
2226	#define PGMPOOL_TD_IDX_MASK 0x3fff
2227	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
2228	* simply too many mappings of this page. */
2229	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
2230
2231	/** @def PGMPOOL_TD_MAKE
2232	* Makes a 16-bit tracking data word.
2233	*
2234	* @returns tracking data.
2235	* @param cRefs The @a cRefs field. Must be within bounds!
2236	* @param idx The @a idx field. Must also be within bounds! */
2237	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
2238
2239	/** @def PGMPOOL_TD_GET_CREFS
2240	* Get the @a cRefs field from a tracking data word.
2241	*
2242	* @returns The @a cRefs field
2243	* @param u16 The tracking data word. */
2244	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
2245
2246	/** @def PGMPOOL_TD_GET_IDX
2247	* Get the @a idx field from a tracking data word.
2248	*
2249	* @returns The @a idx field
2250	* @param u16 The tracking data word. */
2251	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
2252	/** @} */
2253
2254
2255	/**
2256	* Trees are using self relative offsets as pointers.
2257	* So, all its data, including the root pointer, must be in the heap for HC and GC
2258	* to have the same layout.
2259	*/
2260	typedef struct PGMTREES
2261	{
2262	/** Physical access handlers (AVL range+offsetptr tree). */
2263	AVLROGCPHYSTREE PhysHandlers;
2264	/** Virtual access handlers (AVL range + GC ptr tree). */
2265	AVLROGCPTRTREE VirtHandlers;
2266	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
2267	AVLROGCPHYSTREE PhysToVirtHandlers;
2268	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
2269	AVLROGCPTRTREE HyperVirtHandlers;
2270	} PGMTREES;
2271	/** Pointer to PGM trees. */
2272	typedef PGMTREES *PPGMTREES;
2273
2274
2275	/** @name Paging mode macros
2276	* @{ */
2277	#ifdef IN_RC
2278	# define PGM_CTX(a,b) a##RC##b
2279	# define PGM_CTX_STR(a,b) a "GC" b
2280	# define PGM_CTX_DECL(type) VMMRCDECL(type)
2281	#else
2282	# ifdef IN_RING3
2283	# define PGM_CTX(a,b) a##R3##b
2284	# define PGM_CTX_STR(a,b) a "R3" b
2285	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
2286	# else
2287	# define PGM_CTX(a,b) a##R0##b
2288	# define PGM_CTX_STR(a,b) a "R0" b
2289	# define PGM_CTX_DECL(type) VMMDECL(type)
2290	# endif
2291	#endif
2292
2293	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
2294	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
2295	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
2296	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
2297	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
2298	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
2299	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
2300	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
2301	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
2302	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
2303	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
2304	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
2305	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
2306	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
2307	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
2308	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
2309	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
2310
2311	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
2312	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
2313	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
2314	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
2315	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
2316	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
2317	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
2318	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
2319	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
2320	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
2321	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
2322	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2323	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2324	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2325	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2326	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2327	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2328
2329	/* Shw_Gst */
2330	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2331	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2332	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2333	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2334	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2335	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2336	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2337	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2338	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2339	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2340	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2341	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2342	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2343	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2344	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2345	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2346	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2347	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2348	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2349
2350	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2351	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2352	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2353	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2354	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2355	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2356	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2357	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2358	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2359	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2360	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2361	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2362	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2363	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2364	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2365	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2366	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2367	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2368	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2369	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2370	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2371	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2372	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2373	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2374	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2375	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2376	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2377	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2378	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2379	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2380	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2381	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2382	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2383	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2384	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2385	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2386	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2387
2388	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2389	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2390	/** @} */
2391
2392	/**
2393	* Data for each paging mode.
2394	*/
2395	typedef struct PGMMODEDATA
2396	{
2397	/** The guest mode type. */
2398	uint32_t uGstType;
2399	/** The shadow mode type. */
2400	uint32_t uShwType;
2401
2402	/** @name Function pointers for Shadow paging.
2403	* @{
2404	*/
2405	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2406	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2407	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2408	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2409
2410	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2411	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2412
2413	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2414	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2415	/** @} */
2416
2417	/** @name Function pointers for Guest paging.
2418	* @{
2419	*/
2420	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2421	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2422	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2423	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2424	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2425	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2426	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2427	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2428	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2429	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2430	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2431	/** @} */
2432
2433	/** @name Function pointers for Both Shadow and Guest paging.
2434	* @{
2435	*/
2436	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2437	/* no pfnR3BthTrap0eHandler */
2438	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2439	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2440	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2441	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2442	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2443	#ifdef VBOX_STRICT
2444	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2445	#endif
2446	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2447	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2448
2449	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2450	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2451	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2452	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2453	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2454	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2455	#ifdef VBOX_STRICT
2456	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2457	#endif
2458	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2459	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2460
2461	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2462	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2463	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2464	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2465	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2466	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2467	#ifdef VBOX_STRICT
2468	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2469	#endif
2470	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2471	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2472	/** @} */
2473	} PGMMODEDATA, *PPGMMODEDATA;
2474
2475
2476
2477	/**
2478	* Converts a PGM pointer into a VM pointer.
2479	* @returns Pointer to the VM structure the PGM is part of.
2480	* @param pPGM Pointer to PGM instance data.
2481	*/
2482	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2483
2484	/**
2485	* PGM Data (part of VM)
2486	*/
2487	typedef struct PGM
2488	{
2489	/** Offset to the VM structure. */
2490	RTINT offVM;
2491	/** Offset of the PGMCPU structure relative to VMCPU. */
2492	RTINT offVCpuPGM;
2493
2494	/** @cfgm{RamPreAlloc, boolean, false}
2495	* Indicates whether the base RAM should all be allocated before starting
2496	* the VM (default), or if it should be allocated when first written to.
2497	*/
2498	bool fRamPreAlloc;
2499	/** Indicates whether write monitoring is currently in use.
2500	* This is used to prevent conflicts between live saving and page sharing
2501	* detection. */
2502	bool fPhysWriteMonitoringEngaged;
2503	/** Alignment padding. */
2504	bool afAlignment0[2];
2505
2506	/*
2507	* This will be redefined at least two more times before we're done, I'm sure.
2508	* The current code is only to get on with the coding.
2509	* - 2004-06-10: initial version, bird.
2510	* - 2004-07-02: 1st time, bird.
2511	* - 2004-10-18: 2nd time, bird.
2512	* - 2005-07-xx: 3rd time, bird.
2513	*/
2514
2515	/** The host paging mode. (This is what SUPLib reports.) */
2516	SUPPAGINGMODE enmHostMode;
2517
2518	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2519	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
2520	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2521	RCPTRTYPE(PX86PTEPAE) paDynPageMapPaePTEsGC;
2522
2523	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2524	RTGCPHYS GCPhys4MBPSEMask;
2525
2526	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2527	* This is sorted by physical address and contains no overlapping ranges. */
2528	R3PTRTYPE(PPGMRAMRANGE) pRamRangesR3;
2529	/** R0 pointer corresponding to PGM::pRamRangesR3. */
2530	R0PTRTYPE(PPGMRAMRANGE) pRamRangesR0;
2531	/** RC pointer corresponding to PGM::pRamRangesR3. */
2532	RCPTRTYPE(PPGMRAMRANGE) pRamRangesRC;
2533	/** Generation ID for the RAM ranges. This member is incremented everytime a RAM
2534	* range is linked or unlinked. */
2535	uint32_t volatile idRamRangesGen;
2536
2537	/** Pointer to the list of ROM ranges - for R3.
2538	* This is sorted by physical address and contains no overlapping ranges. */
2539	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2540	/** R0 pointer corresponding to PGM::pRomRangesR3. */
2541	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2542	/** RC pointer corresponding to PGM::pRomRangesR3. */
2543	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
2544	#if HC_ARCH_BITS == 64
2545	/** Alignment padding. */
2546	RTRCPTR GCPtrPadding2;
2547	#endif
2548
2549	/** Pointer to the list of MMIO2 ranges - for R3.
2550	* Registration order. */
2551	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
2552
2553	/** PGM offset based trees - R3 Ptr. */
2554	R3PTRTYPE(PPGMTREES) pTreesR3;
2555	/** PGM offset based trees - R0 Ptr. */
2556	R0PTRTYPE(PPGMTREES) pTreesR0;
2557	/** PGM offset based trees - RC Ptr. */
2558	RCPTRTYPE(PPGMTREES) pTreesRC;
2559
2560	/** Linked list of GC mappings - for RC.
2561	* The list is sorted ascending on address.
2562	*/
2563	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
2564	/** Linked list of GC mappings - for HC.
2565	* The list is sorted ascending on address.
2566	*/
2567	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
2568	/** Linked list of GC mappings - for R0.
2569	* The list is sorted ascending on address.
2570	*/
2571	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
2572
2573	/** Pointer to the 5 page CR3 content mapping.
2574	* The first page is always the CR3 (in some form) while the 4 other pages
2575	* are used of the PDs in PAE mode. */
2576	RTGCPTR GCPtrCR3Mapping;
2577	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
2578	uint32_t u32Alignment1;
2579	#endif
2580
2581	/** Indicates that PGMR3FinalizeMappings has been called and that further
2582	* PGMR3MapIntermediate calls will be rejected. */
2583	bool fFinalizedMappings;
2584	/** If set no conflict checks are required. (boolean) */
2585	bool fMappingsFixed;
2586	/** If set, then no mappings are put into the shadow page table. (boolean) */
2587	bool fDisableMappings;
2588	/** Size of fixed mapping */
2589	uint32_t cbMappingFixed;
2590	/** Base address (GC) of fixed mapping */
2591	RTGCPTR GCPtrMappingFixed;
2592	/** The address of the previous RAM range mapping. */
2593	RTGCPTR GCPtrPrevRamRangeMapping;
2594
2595	/** @name Intermediate Context
2596	* @{ */
2597	/** Pointer to the intermediate page directory - Normal. */
2598	R3PTRTYPE(PX86PD) pInterPD;
2599	/** Pointer to the intermedate page tables - Normal.
2600	* There are two page tables, one for the identity mapping and one for
2601	* the host context mapping (of the core code). */
2602	R3PTRTYPE(PX86PT) apInterPTs[2];
2603	/** Pointer to the intermedate page tables - PAE. */
2604	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
2605	/** Pointer to the intermedate page directory - PAE. */
2606	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
2607	/** Pointer to the intermedate page directory - PAE. */
2608	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
2609	/** Pointer to the intermedate page-map level 4 - AMD64. */
2610	R3PTRTYPE(PX86PML4) pInterPaePML4;
2611	/** Pointer to the intermedate page directory - AMD64. */
2612	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
2613	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
2614	RTHCPHYS HCPhysInterPD;
2615	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
2616	RTHCPHYS HCPhysInterPaePDPT;
2617	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
2618	RTHCPHYS HCPhysInterPaePML4;
2619	/** @} */
2620
2621	/** Base address of the dynamic page mapping area.
2622	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
2623	*/
2624	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
2625	/** The index of the last entry used in the dynamic page mapping area. */
2626	RTUINT iDynPageMapLast;
2627	/** Cache containing the last entries in the dynamic page mapping area.
2628	* The cache size is covering half of the mapping area. */
2629	RTHCPHYS aHCPhysDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT + 1)];
2630	/** Keep a lock counter for the full (!) mapping area. */
2631	uint32_t aLockedDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT)];
2632
2633	/** The address of the ring-0 mapping cache if we're making use of it. */
2634	RTR0PTR pvR0DynMapUsed;
2635	#if HC_ARCH_BITS == 32
2636	/** Alignment padding that makes the next member start on a 8 byte boundrary. */
2637	uint32_t u32Alignment2;
2638	#endif
2639
2640	/** PGM critical section.
2641	* This protects the physical & virtual access handlers, ram ranges,
2642	* and the page flag updating (some of it anyway).
2643	*/
2644	PDMCRITSECT CritSect;
2645
2646	/** Pointer to SHW+GST mode data (function pointers).
2647	* The index into this table is made up from */
2648	R3PTRTYPE(PPGMMODEDATA) paModeData;
2649
2650	/** Shadow Page Pool - R3 Ptr. */
2651	R3PTRTYPE(PPGMPOOL) pPoolR3;
2652	/** Shadow Page Pool - R0 Ptr. */
2653	R0PTRTYPE(PPGMPOOL) pPoolR0;
2654	/** Shadow Page Pool - RC Ptr. */
2655	RCPTRTYPE(PPGMPOOL) pPoolRC;
2656
2657	/** We're not in a state which permits writes to guest memory.
2658	* (Only used in strict builds.) */
2659	bool fNoMorePhysWrites;
2660	/** Alignment padding that makes the next member start on a 8 byte boundrary. */
2661	bool afAlignment3[HC_ARCH_BITS == 32 ? 7: 3];
2662
2663	/**
2664	* Data associated with managing the ring-3 mappings of the allocation chunks.
2665	*/
2666	struct
2667	{
2668	/** The chunk tree, ordered by chunk id. */
2669	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2670	R3PTRTYPE(PAVLU32NODECORE) pTree;
2671	#else
2672	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
2673	#endif
2674	/** The chunk age tree, ordered by ageing sequence number. */
2675	R3PTRTYPE(PAVLLU32NODECORE) pAgeTree;
2676	/** The chunk mapping TLB. */
2677	PGMCHUNKR3MAPTLB Tlb;
2678	/** The number of mapped chunks. */
2679	uint32_t c;
2680	/** The maximum number of mapped chunks.
2681	* @cfgm PGM/MaxRing3Chunks */
2682	uint32_t cMax;
2683	/** The current time. */
2684	uint32_t iNow;
2685	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
2686	uint32_t AgeingCountdown;
2687	} ChunkR3Map;
2688
2689	/**
2690	* The page mapping TLB for ring-3 and (for the time being) ring-0.
2691	*/
2692	PGMPAGER3MAPTLB PhysTlbHC;
2693
2694	/** @name The zero page.
2695	* @{ */
2696	/** The host physical address of the zero page. */
2697	RTHCPHYS HCPhysZeroPg;
2698	/** The ring-3 mapping of the zero page. */
2699	RTR3PTR pvZeroPgR3;
2700	/** The ring-0 mapping of the zero page. */
2701	RTR0PTR pvZeroPgR0;
2702	/** The GC mapping of the zero page. */
2703	RTGCPTR pvZeroPgRC;
2704	#if GC_ARCH_BITS != 32
2705	uint32_t u32ZeroAlignment; /*< Alignment padding. /
2706	#endif
2707	/** @}*/
2708
2709	/** The number of handy pages. */
2710	uint32_t cHandyPages;
2711	/**
2712	* Array of handy pages.
2713	*
2714	* This array is used in a two way communication between pgmPhysAllocPage
2715	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
2716	* an intermediary.
2717	*
2718	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
2719	* (The current size of 32 pages, means 128 KB of handy memory.)
2720	*/
2721	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
2722
2723	/**
2724	* Live save data.
2725	*/
2726	struct
2727	{
2728	/** Per type statistics. */
2729	struct
2730	{
2731	/** The number of ready pages. */
2732	uint32_t cReadyPages;
2733	/** The number of dirty pages. */
2734	uint32_t cDirtyPages;
2735	/** The number of ready zero pages. */
2736	uint32_t cZeroPages;
2737	/** The number of write monitored pages. */
2738	uint32_t cMonitoredPages;
2739	} Rom,
2740	Mmio2,
2741	Ram;
2742	/** The number of ignored pages in the RAM ranges (i.e. MMIO, MMIO2 and ROM). */
2743	uint32_t cIgnoredPages;
2744	/** Indicates that a live save operation is active. */
2745	bool fActive;
2746	/** Padding. */
2747	bool afReserved[2];
2748	/** The next history index. */
2749	uint8_t iDirtyPagesHistory;
2750	/** History of the total amount of dirty pages. */
2751	uint32_t acDirtyPagesHistory[64];
2752	/** Short term dirty page average. */
2753	uint32_t cDirtyPagesShort;
2754	/** Long term dirty page average. */
2755	uint32_t cDirtyPagesLong;
2756	/** The number of saved pages. This is used to get some kind of estimate of the
2757	* link speed so we can decide when we're done. It is reset after the first
2758	* 7 passes so the speed estimate doesn't get inflated by the initial set of
2759	* zero pages. */
2760	uint64_t cSavedPages;
2761	/** The nanosecond timestamp when cSavedPages was 0. */
2762	uint64_t uSaveStartNS;
2763	/** Pages per second (for statistics). */
2764	uint32_t cPagesPerSecond;
2765	uint32_t cAlignment;
2766	} LiveSave;
2767
2768	/** @name Error injection.
2769	* @{ */
2770	/** Inject handy page allocation errors pretending we're completely out of
2771	* memory. */
2772	bool volatile fErrInjHandyPages;
2773	/** Padding. */
2774	bool afReserved[7];
2775	/** @} */
2776
2777	/** @name Release Statistics
2778	* @{ */
2779	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero.) /
2780	uint32_t cPrivatePages; /*< The number of private pages. /
2781	uint32_t cSharedPages; /*< The number of shared pages. /
2782	uint32_t cZeroPages; /*< The number of zero backed pages. /
2783	uint32_t cMonitoredPages; /*< The number of write monitored pages. /
2784	uint32_t cWrittenToPages; /*< The number of previously write monitored pages. /
2785	uint32_t cWriteLockedPages; /*< The number of write locked pages. /
2786	uint32_t cReadLockedPages; /*< The number of read locked pages. /
2787
2788	/** The number of times we were forced to change the hypervisor region location. */
2789	STAMCOUNTER cRelocations;
2790	/** @} */
2791
2792	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2793	/* R3 only: */
2794	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2795	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2796
2797	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2798	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2799	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2800	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2801	STAMCOUNTER StatPageMapTlbFlushes; /*< ALL: Ring-3/0 page mapper TLB flushes. /
2802	STAMCOUNTER StatPageMapTlbFlushEntry; /*< ALL: Ring-3/0 page mapper TLB flushes. /
2803	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2804	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2805	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2806	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2807	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2808	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2809	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2810	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2811	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2812	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2813	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2814	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2815	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2816	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2817	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2818	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2819	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2820	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2821
2822	/* RC only: */
2823	STAMCOUNTER StatRCDynMapCacheMisses; /*< RC: The number of dynamic page mapping cache misses /
2824	STAMCOUNTER StatRCDynMapCacheHits; /*< RC: The number of dynamic page mapping cache hits /
2825	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2826	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2827
2828	STAMCOUNTER StatRZPhysRead;
2829	STAMCOUNTER StatRZPhysReadBytes;
2830	STAMCOUNTER StatRZPhysWrite;
2831	STAMCOUNTER StatRZPhysWriteBytes;
2832	STAMCOUNTER StatR3PhysRead;
2833	STAMCOUNTER StatR3PhysReadBytes;
2834	STAMCOUNTER StatR3PhysWrite;
2835	STAMCOUNTER StatR3PhysWriteBytes;
2836	STAMCOUNTER StatRCPhysRead;
2837	STAMCOUNTER StatRCPhysReadBytes;
2838	STAMCOUNTER StatRCPhysWrite;
2839	STAMCOUNTER StatRCPhysWriteBytes;
2840
2841	STAMCOUNTER StatRZPhysSimpleRead;
2842	STAMCOUNTER StatRZPhysSimpleReadBytes;
2843	STAMCOUNTER StatRZPhysSimpleWrite;
2844	STAMCOUNTER StatRZPhysSimpleWriteBytes;
2845	STAMCOUNTER StatR3PhysSimpleRead;
2846	STAMCOUNTER StatR3PhysSimpleReadBytes;
2847	STAMCOUNTER StatR3PhysSimpleWrite;
2848	STAMCOUNTER StatR3PhysSimpleWriteBytes;
2849	STAMCOUNTER StatRCPhysSimpleRead;
2850	STAMCOUNTER StatRCPhysSimpleReadBytes;
2851	STAMCOUNTER StatRCPhysSimpleWrite;
2852	STAMCOUNTER StatRCPhysSimpleWriteBytes;
2853
2854	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2855	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2856	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2857	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2858	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2859	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2860	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2861	# endif
2862	#endif
2863	} PGM;
2864	#ifndef IN_TSTVMSTRUCTGC /* HACK */
2865	AssertCompileMemberAlignment(PGM, paDynPageMap32BitPTEsGC, 8);
2866	AssertCompileMemberAlignment(PGM, GCPtrMappingFixed, sizeof(RTGCPTR));
2867	AssertCompileMemberAlignment(PGM, HCPhysInterPD, 8);
2868	AssertCompileMemberAlignment(PGM, aHCPhysDynPageMapCache, 8);
2869	AssertCompileMemberAlignment(PGM, CritSect, 8);
2870	AssertCompileMemberAlignment(PGM, ChunkR3Map, 8);
2871	AssertCompileMemberAlignment(PGM, PhysTlbHC, 8);
2872	AssertCompileMemberAlignment(PGM, HCPhysZeroPg, 8);
2873	AssertCompileMemberAlignment(PGM, aHandyPages, 8);
2874	AssertCompileMemberAlignment(PGM, cRelocations, 8);
2875	#endif /* !IN_TSTVMSTRUCTGC */
2876	/** Pointer to the PGM instance data. */
2877	typedef PGM *PPGM;
2878
2879
2880	/**
2881	* Converts a PGMCPU pointer into a VM pointer.
2882	* @returns Pointer to the VM structure the PGM is part of.
2883	* @param pPGM Pointer to PGMCPU instance data.
2884	*/
2885	#define PGMCPU2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2886
2887	/**
2888	* Converts a PGMCPU pointer into a PGM pointer.
2889	* @returns Pointer to the VM structure the PGM is part of.
2890	* @param pPGM Pointer to PGMCPU instance data.
2891	*/
2892	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char*)pPGMCpu - pPGMCpu->offPGM) )
2893
2894	/**
2895	* PGMCPU Data (part of VMCPU).
2896	*/
2897	typedef struct PGMCPU
2898	{
2899	/** Offset to the VM structure. */
2900	RTINT offVM;
2901	/** Offset to the VMCPU structure. */
2902	RTINT offVCpu;
2903	/** Offset of the PGM structure relative to VMCPU. */
2904	RTINT offPGM;
2905	RTINT uPadding0; /*< structure size alignment. /
2906
2907	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2908	/** Automatically tracked physical memory mapping set.
2909	* Ring-0 and strict raw-mode builds. */
2910	PGMMAPSET AutoSet;
2911	#endif
2912
2913	/** A20 gate mask.
2914	* Our current approach to A20 emulation is to let REM do it and don't bother
2915	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2916	* But whould need arrise, we'll subject physical addresses to this mask. */
2917	RTGCPHYS GCPhysA20Mask;
2918	/** A20 gate state - boolean! */
2919	bool fA20Enabled;
2920
2921	/** What needs syncing (PGM_SYNC_*).
2922	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2923	* PGMFlushTLB, and PGMR3Load. */
2924	RTUINT fSyncFlags;
2925
2926	/** The shadow paging mode. */
2927	PGMMODE enmShadowMode;
2928	/** The guest paging mode. */
2929	PGMMODE enmGuestMode;
2930
2931	/** The current physical address representing in the guest CR3 register. */
2932	RTGCPHYS GCPhysCR3;
2933
2934	/** @name 32-bit Guest Paging.
2935	* @{ */
2936	/** The guest's page directory, R3 pointer. */
2937	R3PTRTYPE(PX86PD) pGst32BitPdR3;
2938	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2939	/** The guest's page directory, R0 pointer. */
2940	R0PTRTYPE(PX86PD) pGst32BitPdR0;
2941	#endif
2942	/** The guest's page directory, static RC mapping. */
2943	RCPTRTYPE(PX86PD) pGst32BitPdRC;
2944	/** @} */
2945
2946	/** @name PAE Guest Paging.
2947	* @{ */
2948	/** The guest's page directory pointer table, static RC mapping. */
2949	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
2950	/** The guest's page directory pointer table, R3 pointer. */
2951	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
2952	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2953	/** The guest's page directory pointer table, R0 pointer. */
2954	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
2955	#endif
2956
2957	/** The guest's page directories, R3 pointers.
2958	* These are individual pointers and don't have to be adjecent.
2959	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2960	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
2961	/** The guest's page directories, R0 pointers.
2962	* Same restrictions as apGstPaePDsR3. */
2963	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2964	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
2965	#endif
2966	/** The guest's page directories, static GC mapping.
2967	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
2968	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2969	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
2970	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
2971	RTGCPHYS aGCPhysGstPaePDs[4];
2972	/** The physical addresses of the monitored guest page directories (PAE). */
2973	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
2974	/** @} */
2975
2976	/** @name AMD64 Guest Paging.
2977	* @{ */
2978	/** The guest's page directory pointer table, R3 pointer. */
2979	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
2980	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2981	/** The guest's page directory pointer table, R0 pointer. */
2982	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
2983	#else
2984	RTR0PTR alignment6b; /*< alignment equalizer. /
2985	#endif
2986	/** @} */
2987
2988	/** Pointer to the page of the current active CR3 - R3 Ptr. */
2989	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
2990	/** Pointer to the page of the current active CR3 - R0 Ptr. */
2991	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
2992	/** Pointer to the page of the current active CR3 - RC Ptr. */
2993	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
2994	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
2995	uint32_t iShwUser;
2996	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
2997	uint32_t iShwUserTable;
2998	# if HC_ARCH_BITS == 64
2999	RTRCPTR alignment6; /*< structure size alignment. /
3000	# endif
3001	/** @} */
3002
3003	/** @name Function pointers for Shadow paging.
3004	* @{
3005	*/
3006	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3007	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
3008	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3009	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3010
3011	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3012	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3013
3014	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
3015	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3016
3017	/** @} */
3018
3019	/** @name Function pointers for Guest paging.
3020	* @{
3021	*/
3022	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3023	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
3024	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3025	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3026	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3027	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3028	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3029	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3030	#if HC_ARCH_BITS == 64
3031	RTRCPTR alignment3; /*< structure size alignment. /
3032	#endif
3033
3034	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
3035	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
3036	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
3037	/** @} */
3038
3039	/** @name Function pointers for Both Shadow and Guest paging.
3040	* @{
3041	*/
3042	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
3043	/* no pfnR3BthTrap0eHandler */
3044	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3045	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3046	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
3047	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3048	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3049	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3050	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3051	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
3052
3053	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
3054	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3055	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3056	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
3057	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3058	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3059	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3060	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3061	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
3062
3063	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
3064	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3065	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
3066	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
3067	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
3068	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
3069	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
3070	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
3071	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
3072	RTRCPTR alignment2; /*< structure size alignment. /
3073	/** @} */
3074
3075	/** For saving stack space, the disassembler state is allocated here instead of
3076	* on the stack.
3077	* @note The DISCPUSTATE structure is not R3/R0/RZ clean! */
3078	union
3079	{
3080	/** The disassembler scratch space. */
3081	DISCPUSTATE DisState;
3082	/** Padding. */
3083	uint8_t abDisStatePadding[DISCPUSTATE_PADDING_SIZE];
3084	};
3085
3086	/* Count the number of pgm pool access handler calls. */
3087	uint64_t cPoolAccessHandler;
3088
3089	/** @name Release Statistics
3090	* @{ */
3091	/** The number of times the guest has switched mode since last reset or statistics reset. */
3092	STAMCOUNTER cGuestModeChanges;
3093	/** @} */
3094
3095	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
3096	/** @name Statistics
3097	* @{ */
3098	/** RC: Which statistic this \#PF should be attributed to. */
3099	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
3100	RTRCPTR padding0;
3101	/** R0: Which statistic this \#PF should be attributed to. */
3102	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
3103	RTR0PTR padding1;
3104
3105	/* Common */
3106	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
3107	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
3108
3109	/* R0 only: */
3110	STAMCOUNTER StatR0DynMapMigrateInvlPg; /*< R0: invlpg in PGMDynMapMigrateAutoSet. /
3111	STAMPROFILE StatR0DynMapGCPageInl; /*< R0: Calls to pgmR0DynMapGCPageInlined. /
3112	STAMCOUNTER StatR0DynMapGCPageInlHits; /*< R0: Hash table lookup hits. /
3113	STAMCOUNTER StatR0DynMapGCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
3114	STAMCOUNTER StatR0DynMapGCPageInlRamHits; /*< R0: 1st ram range hits. /
3115	STAMCOUNTER StatR0DynMapGCPageInlRamMisses; /*< R0: 1st ram range misses, takes slow path. /
3116	STAMPROFILE StatR0DynMapHCPageInl; /*< R0: Calls to pgmR0DynMapHCPageInlined. /
3117	STAMCOUNTER StatR0DynMapHCPageInlHits; /*< R0: Hash table lookup hits. /
3118	STAMCOUNTER StatR0DynMapHCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
3119	STAMPROFILE StatR0DynMapHCPage; /*< R0: Calls to PGMDynMapHCPage. /
3120	STAMCOUNTER StatR0DynMapSetOptimize; /*< R0: Calls to pgmDynMapOptimizeAutoSet. /
3121	STAMCOUNTER StatR0DynMapSetSearchFlushes; /*< R0: Set search restorting to subset flushes. /
3122	STAMCOUNTER StatR0DynMapSetSearchHits; /*< R0: Set search hits. /
3123	STAMCOUNTER StatR0DynMapSetSearchMisses; /*< R0: Set search misses. /
3124	STAMCOUNTER StatR0DynMapPage; /*< R0: Calls to pgmR0DynMapPage. /
3125	STAMCOUNTER StatR0DynMapPageHits0; /*< R0: Hits at iPage+0. /
3126	STAMCOUNTER StatR0DynMapPageHits1; /*< R0: Hits at iPage+1. /
3127	STAMCOUNTER StatR0DynMapPageHits2; /*< R0: Hits at iPage+2. /
3128	STAMCOUNTER StatR0DynMapPageInvlPg; /*< R0: invlpg. /
3129	STAMCOUNTER StatR0DynMapPageSlow; /*< R0: Calls to pgmR0DynMapPageSlow. /
3130	STAMCOUNTER StatR0DynMapPageSlowLoopHits; /*< R0: Hits in the pgmR0DynMapPageSlow search loop. /
3131	STAMCOUNTER StatR0DynMapPageSlowLoopMisses; /*< R0: Misses in the pgmR0DynMapPageSlow search loop. /
3132	//STAMCOUNTER StatR0DynMapPageSlowLostHits; /*< R0: Lost hits. /
3133	STAMCOUNTER StatR0DynMapSubsets; /*< R0: Times PGMDynMapPushAutoSubset was called. /
3134	STAMCOUNTER StatR0DynMapPopFlushes; /*< R0: Times PGMDynMapPopAutoSubset flushes the subset. /
3135	STAMCOUNTER aStatR0DynMapSetSize[11]; /*< R0: Set size distribution. /
3136
3137	/* RZ only: */
3138	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
3139	STAMPROFILE StatRZTrap0eTimeCheckPageFault;
3140	STAMPROFILE StatRZTrap0eTimeSyncPT;
3141	STAMPROFILE StatRZTrap0eTimeMapping;
3142	STAMPROFILE StatRZTrap0eTimeOutOfSync;
3143	STAMPROFILE StatRZTrap0eTimeHandlers;
3144	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
3145	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
3146	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
3147	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
3148	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
3149	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
3150	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
3151	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
3152	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
3153	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
3154	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
3155	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
3156	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
3157	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
3158	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
3159	STAMCOUNTER StatRZTrap0eHandlersPhysical; /*< RC/R0: Number of traps due to physical access handlers. /
3160	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
3161	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
3162	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
3163	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
3164	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
3165	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: #PF err kind /
3166	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: #PF err kind /
3167	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: #PF err kind /
3168	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: #PF err kind /
3169	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: #PF err kind /
3170	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: #PF err kind /
3171	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: #PF err kind /
3172	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: #PF err kind /
3173	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: #PF err kind /
3174	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: #PF err kind /
3175	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: #PF err kind /
3176	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest #PFs. /
3177	STAMCOUNTER StatRZTrap0eGuestPFUnh; /*< RC/R0: Real guest #PF ending up at the end of the #PF code. /
3178	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest #PF to HMA or other mapping. /
3179	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
3180	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
3181	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the #PFs. /
3182	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
3183	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
3184	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
3185	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
3186	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
3187
3188	/* HC - R3 and (maybe) R0: */
3189
3190	/* RZ & R3: */
3191	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
3192	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
3193	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
3194	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
3195	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
3196	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
3197	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
3198	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
3199	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
3200	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
3201	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
3202	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
3203	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
3204	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
3205	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
3206	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
3207	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
3208	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
3209	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
3210	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
3211	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
3212	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
3213	STAMCOUNTER StatRZDirtyPageStale; /*< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) /
3214	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
3215	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
3216	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
3217	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
3218	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
3219	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
3220	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
3221	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
3222	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
3223	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
3224	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
3225	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
3226	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
3227	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
3228	STAMCOUNTER StatRZPageOutOfSyncUserWrite; /*< RC/R0: The number of times user page is out of sync was detected in #PF. /
3229	STAMCOUNTER StatRZPageOutOfSyncSupervisorWrite; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF. /
3230	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
3231	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
3232	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
3233	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
3234	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
3235	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
3236	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
3237
3238	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
3239	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
3240	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
3241	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
3242	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
3243	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
3244	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
3245	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
3246	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
3247	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
3248	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
3249	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
3250	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
3251	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
3252	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
3253	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
3254	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
3255	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
3256	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
3257	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
3258	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
3259	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
3260	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
3261	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
3262	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
3263	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
3264	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
3265	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
3266	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
3267	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
3268	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
3269	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
3270	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
3271	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
3272	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
3273	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
3274	STAMCOUNTER StatR3PageOutOfSyncUserWrite; /*< R3: The number of times user page is out of sync was detected in #PF. /
3275	STAMCOUNTER StatR3PageOutOfSyncSupervisorWrite; /*< R3: The number of times supervisor page is out of sync was detected in in #PF. /
3276	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
3277	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
3278	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
3279	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
3280	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
3281	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
3282	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
3283	/** @} */
3284	#endif /* VBOX_WITH_STATISTICS */
3285	} PGMCPU;
3286	/** Pointer to the per-cpu PGM data. */
3287	typedef PGMCPU *PPGMCPU;
3288
3289
3290	/** @name PGM::fSyncFlags Flags
3291	* @{
3292	*/
3293	/** Updates the virtual access handler state bit in PGMPAGE. */
3294	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
3295	/** Always sync CR3. */
3296	#define PGM_SYNC_ALWAYS RT_BIT(1)
3297	/** Check monitoring on next CR3 (re)load and invalidate page.
3298	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
3299	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
3300	/** Check guest mapping in SyncCR3. */
3301	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
3302	/** Clear the page pool (a light weight flush). */
3303	#define PGM_SYNC_CLEAR_PGM_POOL_BIT 8
3304	#define PGM_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_SYNC_CLEAR_PGM_POOL_BIT)
3305	/** @} */
3306
3307
3308	RT_C_DECLS_BEGIN
3309
3310	int pgmLock(PVM pVM);
3311	void pgmUnlock(PVM pVM);
3312
3313	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
3314	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
3315	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
3316	void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping);
3317	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3318
3319	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
3320	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
3321	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage);
3322	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
3323	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
3324	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
3325	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
3326	#else
3327	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
3328	#endif
3329	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
3330	int pgmR3InitSavedState(PVM pVM, uint64_t cbRam);
3331
3332	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3333	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
3334	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3335	void pgmPhysPageMakeWriteMonitoredWritable(PVM pVM, PPGMPAGE pPage);
3336	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3337	int pgmPhysPageMakeWritableUnlocked(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
3338	int pgmPhysPageMakeWritableAndMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3339	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3340	int pgmPhysPageMapReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv);
3341	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
3342	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
3343	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
3344	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
3345	#ifdef IN_RING3
3346	void pgmR3PhysRelinkRamRanges(PVM pVM);
3347	int pgmR3PhysRamPreAllocate(PVM pVM);
3348	int pgmR3PhysRamReset(PVM pVM);
3349	int pgmR3PhysRomReset(PVM pVM);
3350	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
3351
3352	int pgmR3PoolInit(PVM pVM);
3353	void pgmR3PoolRelocate(PVM pVM);
3354	void pgmR3PoolReset(PVM pVM);
3355	void pgmR3PoolClearAll(PVM pVM);
3356
3357	#endif /* IN_RING3 */
3358	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3359	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
3360	#endif
3361	int pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage, bool fLockPage = false);
3362
3363	DECLINLINE(int) pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage, bool fLockPage = false)
3364	{
3365	return pgmPoolAllocEx(pVM, GCPhys, enmKind, PGMPOOLACCESS_DONTCARE, iUser, iUserTable, ppPage, fLockPage);
3366	}
3367
3368	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
3369	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
3370	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3371	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys);
3372	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
3373	int pgmPoolSyncCR3(PVMCPU pVCpu);
3374	bool pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys);
3375	int pgmPoolTrackUpdateGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs);
3376	DECLINLINE(int) pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs)
3377	{
3378	return pgmPoolTrackUpdateGCPhys(pVM, pPhysPage, true /* flush PTEs */, pfFlushTLBs);
3379	}
3380
3381	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT);
3382	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage);
3383	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint);
3384	#ifdef PGMPOOL_WITH_MONITORING
3385	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu);
3386	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3387	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3388	#endif
3389
3390	void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage);
3391	void pgmPoolResetDirtyPages(PVM pVM);
3392
3393	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
3394	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
3395
3396	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
3397	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
3398	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
3399	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
3400
3401	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
3402	#ifndef IN_RC
3403	int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
3404	#endif
3405	int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
3406
3407	PX86PD pgmGstLazyMap32BitPD(PPGMCPU pPGM);
3408	PX86PDPT pgmGstLazyMapPaePDPT(PPGMCPU pPGM);
3409	PX86PDPAE pgmGstLazyMapPaePD(PPGMCPU pPGM, uint32_t iPdpt);
3410	PX86PML4 pgmGstLazyMapPml4(PPGMCPU pPGM);
3411
3412	RT_C_DECLS_END
3413
3414
3415	/**
3416	* Gets the PGMRAMRANGE structure for a guest page.
3417	*
3418	* @returns Pointer to the RAM range on success.
3419	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3420	*
3421	* @param pPGM PGM handle.
3422	* @param GCPhys The GC physical address.
3423	*/
3424	DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
3425	{
3426	/*
3427	* Optimize for the first range.
3428	*/
3429	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3430	RTGCPHYS off = GCPhys - pRam->GCPhys;
3431	if (RT_UNLIKELY(off >= pRam->cb))
3432	{
3433	do
3434	{
3435	pRam = pRam->CTX_SUFF(pNext);
3436	if (RT_UNLIKELY(!pRam))
3437	break;
3438	off = GCPhys - pRam->GCPhys;
3439	} while (off >= pRam->cb);
3440	}
3441	return pRam;
3442	}
3443
3444
3445	/**
3446	* Gets the PGMPAGE structure for a guest page.
3447	*
3448	* @returns Pointer to the page on success.
3449	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3450	*
3451	* @param pPGM PGM handle.
3452	* @param GCPhys The GC physical address.
3453	*/
3454	DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
3455	{
3456	/*
3457	* Optimize for the first range.
3458	*/
3459	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3460	RTGCPHYS off = GCPhys - pRam->GCPhys;
3461	if (RT_UNLIKELY(off >= pRam->cb))
3462	{
3463	do
3464	{
3465	pRam = pRam->CTX_SUFF(pNext);
3466	if (RT_UNLIKELY(!pRam))
3467	return NULL;
3468	off = GCPhys - pRam->GCPhys;
3469	} while (off >= pRam->cb);
3470	}
3471	return &pRam->aPages[off >> PAGE_SHIFT];
3472	}
3473
3474
3475	/**
3476	* Gets the PGMPAGE structure for a guest page.
3477	*
3478	* Old Phys code: Will make sure the page is present.
3479	*
3480	* @returns VBox status code.
3481	* @retval VINF_SUCCESS and a valid *ppPage on success.
3482	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3483	*
3484	* @param pPGM PGM handle.
3485	* @param GCPhys The GC physical address.
3486	* @param ppPage Where to store the page pointer on success.
3487	*/
3488	DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
3489	{
3490	/*
3491	* Optimize for the first range.
3492	*/
3493	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3494	RTGCPHYS off = GCPhys - pRam->GCPhys;
3495	if (RT_UNLIKELY(off >= pRam->cb))
3496	{
3497	do
3498	{
3499	pRam = pRam->CTX_SUFF(pNext);
3500	if (RT_UNLIKELY(!pRam))
3501	{
3502	ppPage = NULL; / avoid incorrect and very annoying GCC warnings */
3503	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3504	}
3505	off = GCPhys - pRam->GCPhys;
3506	} while (off >= pRam->cb);
3507	}
3508	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3509	return VINF_SUCCESS;
3510	}
3511
3512
3513
3514
3515	/**
3516	* Gets the PGMPAGE structure for a guest page.
3517	*
3518	* Old Phys code: Will make sure the page is present.
3519	*
3520	* @returns VBox status code.
3521	* @retval VINF_SUCCESS and a valid *ppPage on success.
3522	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3523	*
3524	* @param pPGM PGM handle.
3525	* @param GCPhys The GC physical address.
3526	* @param ppPage Where to store the page pointer on success.
3527	* @param ppRamHint Where to read and store the ram list hint.
3528	* The caller initializes this to NULL before the call.
3529	*/
3530	DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
3531	{
3532	RTGCPHYS off;
3533	PPGMRAMRANGE pRam = *ppRamHint;
3534	if ( !pRam
3535	\|\| RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
3536	{
3537	pRam = pPGM->CTX_SUFF(pRamRanges);
3538	off = GCPhys - pRam->GCPhys;
3539	if (RT_UNLIKELY(off >= pRam->cb))
3540	{
3541	do
3542	{
3543	pRam = pRam->CTX_SUFF(pNext);
3544	if (RT_UNLIKELY(!pRam))
3545	{
3546	ppPage = NULL; / Kill the incorrect and extremely annoying GCC warnings. */
3547	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3548	}
3549	off = GCPhys - pRam->GCPhys;
3550	} while (off >= pRam->cb);
3551	}
3552	*ppRamHint = pRam;
3553	}
3554	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3555	return VINF_SUCCESS;
3556	}
3557
3558
3559	/**
3560	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3561	*
3562	* @returns Pointer to the page on success.
3563	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3564	*
3565	* @param pPGM PGM handle.
3566	* @param GCPhys The GC physical address.
3567	* @param ppRam Where to store the pointer to the PGMRAMRANGE.
3568	*/
3569	DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
3570	{
3571	/*
3572	* Optimize for the first range.
3573	*/
3574	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3575	RTGCPHYS off = GCPhys - pRam->GCPhys;
3576	if (RT_UNLIKELY(off >= pRam->cb))
3577	{
3578	do
3579	{
3580	pRam = pRam->CTX_SUFF(pNext);
3581	if (RT_UNLIKELY(!pRam))
3582	return NULL;
3583	off = GCPhys - pRam->GCPhys;
3584	} while (off >= pRam->cb);
3585	}
3586	*ppRam = pRam;
3587	return &pRam->aPages[off >> PAGE_SHIFT];
3588	}
3589
3590
3591	/**
3592	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3593	*
3594	* @returns Pointer to the page on success.
3595	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3596	*
3597	* @param pPGM PGM handle.
3598	* @param GCPhys The GC physical address.
3599	* @param ppPage Where to store the pointer to the PGMPAGE structure.
3600	* @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
3601	*/
3602	DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
3603	{
3604	/*
3605	* Optimize for the first range.
3606	*/
3607	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3608	RTGCPHYS off = GCPhys - pRam->GCPhys;
3609	if (RT_UNLIKELY(off >= pRam->cb))
3610	{
3611	do
3612	{
3613	pRam = pRam->CTX_SUFF(pNext);
3614	if (RT_UNLIKELY(!pRam))
3615	{
3616	ppRam = NULL; / Shut up silly GCC warnings. */
3617	ppPage = NULL; / ditto */
3618	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3619	}
3620	off = GCPhys - pRam->GCPhys;
3621	} while (off >= pRam->cb);
3622	}
3623	*ppRam = pRam;
3624	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3625	return VINF_SUCCESS;
3626	}
3627
3628
3629	/**
3630	* Convert GC Phys to HC Phys.
3631	*
3632	* @returns VBox status.
3633	* @param pPGM PGM handle.
3634	* @param GCPhys The GC physical address.
3635	* @param pHCPhys Where to store the corresponding HC physical address.
3636	*
3637	* @deprecated Doesn't deal with zero, shared or write monitored pages.
3638	* Avoid when writing new code!
3639	*/
3640	DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
3641	{
3642	PPGMPAGE pPage;
3643	int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
3644	if (RT_FAILURE(rc))
3645	return rc;
3646	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
3647	return VINF_SUCCESS;
3648	}
3649
3650	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3651
3652	/**
3653	* Inlined version of the ring-0 version of PGMDynMapHCPage that
3654	* optimizes access to pages already in the set.
3655	*
3656	* @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
3657	* @param pPGM Pointer to the PVM instance data.
3658	* @param HCPhys The physical address of the page.
3659	* @param ppv Where to store the mapping address.
3660	*/
3661	DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
3662	{
3663	PVM pVM = PGM2VM(pPGM);
3664	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3665	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3666
3667	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
3668	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3669	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3670
3671	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3672	unsigned iEntry = pSet->aiHashTable[iHash];
3673	if ( iEntry < pSet->cEntries
3674	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3675	{
3676	*ppv = pSet->aEntries[iEntry].pvPage;
3677	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
3678	}
3679	else
3680	{
3681	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
3682	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3683	}
3684
3685	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
3686	return VINF_SUCCESS;
3687	}
3688
3689
3690	/**
3691	* Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
3692	* access to pages already in the set.
3693	*
3694	* @returns See PGMDynMapGCPage.
3695	* @param pPGM Pointer to the PVM instance data.
3696	* @param GCPhys The guest physical address of the page.
3697	* @param ppv Where to store the mapping address.
3698	*/
3699	DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3700	{
3701	PVM pVM = PGM2VM(pPGM);
3702	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3703
3704	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3705	AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("%RGp\n", GCPhys));
3706
3707	/*
3708	* Get the ram range.
3709	*/
3710	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3711	RTGCPHYS off = GCPhys - pRam->GCPhys;
3712	if (RT_UNLIKELY(off >= pRam->cb
3713	/** @todo \|\| page state stuff */))
3714	{
3715	/* This case is not counted into StatR0DynMapGCPageInl. */
3716	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3717	return PGMDynMapGCPage(pVM, GCPhys, ppv);
3718	}
3719
3720	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3721	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3722
3723	/*
3724	* pgmR0DynMapHCPageInlined with out stats.
3725	*/
3726	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3727	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3728	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3729
3730	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3731	unsigned iEntry = pSet->aiHashTable[iHash];
3732	if ( iEntry < pSet->cEntries
3733	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3734	{
3735	*ppv = pSet->aEntries[iEntry].pvPage;
3736	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3737	}
3738	else
3739	{
3740	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3741	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3742	}
3743
3744	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3745	return VINF_SUCCESS;
3746	}
3747
3748
3749	/**
3750	* Inlined version of the ring-0 version of PGMDynMapGCPageOff that optimizes
3751	* access to pages already in the set.
3752	*
3753	* @returns See PGMDynMapGCPage.
3754	* @param pPGM Pointer to the PVM instance data.
3755	* @param HCPhys The physical address of the page.
3756	* @param ppv Where to store the mapping address.
3757	*/
3758	DECLINLINE(int) pgmR0DynMapGCPageOffInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3759	{
3760	PVM pVM = PGM2VM(pPGM);
3761	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3762
3763	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3764
3765	/*
3766	* Get the ram range.
3767	*/
3768	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3769	RTGCPHYS off = GCPhys - pRam->GCPhys;
3770	if (RT_UNLIKELY(off >= pRam->cb
3771	/** @todo \|\| page state stuff */))
3772	{
3773	/* This case is not counted into StatR0DynMapGCPageInl. */
3774	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3775	return PGMDynMapGCPageOff(pVM, GCPhys, ppv);
3776	}
3777
3778	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3779	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3780
3781	/*
3782	* pgmR0DynMapHCPageInlined with out stats.
3783	*/
3784	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3785	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3786	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3787
3788	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3789	unsigned iEntry = pSet->aiHashTable[iHash];
3790	if ( iEntry < pSet->cEntries
3791	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3792	{
3793	ppv = (void )((uintptr_t)pSet->aEntries[iEntry].pvPage \| (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
3794	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3795	}
3796	else
3797	{
3798	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3799	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3800	ppv = (void )((uintptr_t)*ppv \| (PAGE_OFFSET_MASK & (uintptr_t)GCPhys));
3801	}
3802
3803	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3804	return VINF_SUCCESS;
3805	}
3806
3807	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3808	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3809
3810	/**
3811	* Maps the page into current context (RC and maybe R0).
3812	*
3813	* @returns pointer to the mapping.
3814	* @param pVM Pointer to the PGM instance data.
3815	* @param pPage The page.
3816	*/
3817	DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
3818	{
3819	if (pPage->idx >= PGMPOOL_IDX_FIRST)
3820	{
3821	Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
3822	void *pv;
3823	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3824	pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
3825	# else
3826	PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
3827	# endif
3828	return pv;
3829	}
3830	AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
3831	}
3832
3833	/**
3834	* Temporarily maps one host page specified by HC physical address, returning
3835	* pointer within the page.
3836	*
3837	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
3838	* reused after 8 mappings (or perhaps a few more if you score with the cache).
3839	*
3840	* @returns The address corresponding to HCPhys.
3841	* @param pPGM Pointer to the PVM instance data.
3842	* @param HCPhys HC Physical address of the page.
3843	*/
3844	DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
3845	{
3846	void *pv;
3847	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3848	pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3849	# else
3850	PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3851	# endif
3852	pv = (void *)((uintptr_t)pv \| ((uintptr_t)HCPhys & PAGE_OFFSET_MASK));
3853	return pv;
3854	}
3855
3856	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 \|\| IN_RC */
3857	#ifndef IN_RC
3858
3859	/**
3860	* Queries the Physical TLB entry for a physical guest page,
3861	* attempting to load the TLB entry if necessary.
3862	*
3863	* @returns VBox status code.
3864	* @retval VINF_SUCCESS on success
3865	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3866	*
3867	* @param pPGM The PGM instance handle.
3868	* @param GCPhys The address of the guest page.
3869	* @param ppTlbe Where to store the pointer to the TLB entry.
3870	*/
3871	DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3872	{
3873	int rc;
3874	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3875	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3876	{
3877	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3878	rc = VINF_SUCCESS;
3879	}
3880	else
3881	rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
3882	*ppTlbe = pTlbe;
3883	return rc;
3884	}
3885
3886
3887	/**
3888	* Queries the Physical TLB entry for a physical guest page,
3889	* attempting to load the TLB entry if necessary.
3890	*
3891	* @returns VBox status code.
3892	* @retval VINF_SUCCESS on success
3893	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3894	*
3895	* @param pPGM The PGM instance handle.
3896	* @param pPage Pointer to the PGMPAGE structure corresponding to
3897	* GCPhys.
3898	* @param GCPhys The address of the guest page.
3899	* @param ppTlbe Where to store the pointer to the TLB entry.
3900	*/
3901	DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3902	{
3903	int rc;
3904	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3905	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3906	{
3907	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3908	rc = VINF_SUCCESS;
3909	}
3910	else
3911	rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
3912	*ppTlbe = pTlbe;
3913	return rc;
3914	}
3915
3916	#endif /* !IN_RC */
3917
3918	/**
3919	* Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
3920	* Takes PSE-36 into account.
3921	*
3922	* @returns guest physical address
3923	* @param pPGM Pointer to the PGM instance data.
3924	* @param Pde Guest Pde
3925	*/
3926	DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
3927	{
3928	RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
3929	GCPhys \|= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
3930
3931	return GCPhys & pPGM->GCPhys4MBPSEMask;
3932	}
3933
3934
3935	/**
3936	* Gets the page directory entry for the specified address (32-bit paging).
3937	*
3938	* @returns The page directory entry in question.
3939	* @param pPGM Pointer to the PGM instance data.
3940	* @param GCPtr The address.
3941	*/
3942	DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3943	{
3944	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3945	PCX86PD pGuestPD = NULL;
3946	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3947	if (RT_FAILURE(rc))
3948	{
3949	X86PDE ZeroPde = {0};
3950	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
3951	}
3952	#else
3953	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3954	# ifdef IN_RING3
3955	if (!pGuestPD)
3956	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3957	# endif
3958	#endif
3959	return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3960	}
3961
3962
3963	/**
3964	* Gets the address of a specific page directory entry (32-bit paging).
3965	*
3966	* @returns Pointer the page directory entry in question.
3967	* @param pPGM Pointer to the PGM instance data.
3968	* @param GCPtr The address.
3969	*/
3970	DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3971	{
3972	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3973	PX86PD pGuestPD = NULL;
3974	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3975	AssertRCReturn(rc, NULL);
3976	#else
3977	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3978	# ifdef IN_RING3
3979	if (!pGuestPD)
3980	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3981	# endif
3982	#endif
3983	return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3984	}
3985
3986
3987	/**
3988	* Gets the address the guest page directory (32-bit paging).
3989	*
3990	* @returns Pointer the page directory entry in question.
3991	* @param pPGM Pointer to the PGM instance data.
3992	*/
3993	DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
3994	{
3995	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3996	PX86PD pGuestPD = NULL;
3997	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3998	AssertRCReturn(rc, NULL);
3999	#else
4000	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
4001	# ifdef IN_RING3
4002	if (!pGuestPD)
4003	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
4004	# endif
4005	#endif
4006	return pGuestPD;
4007	}
4008
4009
4010	/**
4011	* Gets the guest page directory pointer table.
4012	*
4013	* @returns Pointer to the page directory in question.
4014	* @returns NULL if the page directory is not present or on an invalid page.
4015	* @param pPGM Pointer to the PGM instance data.
4016	*/
4017	DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
4018	{
4019	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4020	PX86PDPT pGuestPDPT = NULL;
4021	int rc = pgmR0DynMapGCPageOffInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
4022	AssertRCReturn(rc, NULL);
4023	#else
4024	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
4025	# ifdef IN_RING3
4026	if (!pGuestPDPT)
4027	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
4028	# endif
4029	#endif
4030	return pGuestPDPT;
4031	}
4032
4033
4034	/**
4035	* Gets the guest page directory pointer table entry for the specified address.
4036	*
4037	* @returns Pointer to the page directory in question.
4038	* @returns NULL if the page directory is not present or on an invalid page.
4039	* @param pPGM Pointer to the PGM instance data.
4040	* @param GCPtr The address.
4041	*/
4042	DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4043	{
4044	AssertGCPtr32(GCPtr);
4045
4046	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4047	PX86PDPT pGuestPDPT = 0;
4048	int rc = pgmR0DynMapGCPageOffInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
4049	AssertRCReturn(rc, 0);
4050	#else
4051	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
4052	# ifdef IN_RING3
4053	if (!pGuestPDPT)
4054	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
4055	# endif
4056	#endif
4057	return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
4058	}
4059
4060
4061	/**
4062	* Gets the page directory for the specified address.
4063	*
4064	* @returns Pointer to the page directory in question.
4065	* @returns NULL if the page directory is not present or on an invalid page.
4066	* @param pPGM Pointer to the PGM instance data.
4067	* @param GCPtr The address.
4068	*/
4069	DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
4070	{
4071	AssertGCPtr32(GCPtr);
4072
4073	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
4074	AssertReturn(pGuestPDPT, NULL);
4075	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4076	if (pGuestPDPT->a[iPdpt].n.u1Present)
4077	{
4078	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4079	PX86PDPAE pGuestPD = NULL;
4080	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
4081	AssertRCReturn(rc, NULL);
4082	#else
4083	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
4084	if ( !pGuestPD
4085	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
4086	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
4087	#endif
4088	return pGuestPD;
4089	/* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
4090	}
4091	return NULL;
4092	}
4093
4094
4095	/**
4096	* Gets the page directory entry for the specified address.
4097	*
4098	* @returns Pointer to the page directory entry in question.
4099	* @returns NULL if the page directory is not present or on an invalid page.
4100	* @param pPGM Pointer to the PGM instance data.
4101	* @param GCPtr The address.
4102	*/
4103	DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4104	{
4105	AssertGCPtr32(GCPtr);
4106
4107	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
4108	AssertReturn(pGuestPDPT, NULL);
4109	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4110	if (pGuestPDPT->a[iPdpt].n.u1Present)
4111	{
4112	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4113	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4114	PX86PDPAE pGuestPD = NULL;
4115	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
4116	AssertRCReturn(rc, NULL);
4117	#else
4118	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
4119	if ( !pGuestPD
4120	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
4121	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
4122	#endif
4123	return &pGuestPD->a[iPD];
4124	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
4125	}
4126	return NULL;
4127	}
4128
4129
4130	/**
4131	* Gets the page directory entry for the specified address.
4132	*
4133	* @returns The page directory entry in question.
4134	* @returns A non-present entry if the page directory is not present or on an invalid page.
4135	* @param pPGM Pointer to the PGM instance data.
4136	* @param GCPtr The address.
4137	*/
4138	DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4139	{
4140	AssertGCPtr32(GCPtr);
4141	X86PDEPAE ZeroPde = {0};
4142	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
4143	if (RT_LIKELY(pGuestPDPT))
4144	{
4145	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4146	if (pGuestPDPT->a[iPdpt].n.u1Present)
4147	{
4148	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4149	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4150	PX86PDPAE pGuestPD = NULL;
4151	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
4152	AssertRCReturn(rc, ZeroPde);
4153	#else
4154	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
4155	if ( !pGuestPD
4156	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
4157	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
4158	#endif
4159	return pGuestPD->a[iPD];
4160	}
4161	}
4162	return ZeroPde;
4163	}
4164
4165
4166	/**
4167	* Gets the page directory pointer table entry for the specified address
4168	* and returns the index into the page directory
4169	*
4170	* @returns Pointer to the page directory in question.
4171	* @returns NULL if the page directory is not present or on an invalid page.
4172	* @param pPGM Pointer to the PGM instance data.
4173	* @param GCPtr The address.
4174	* @param piPD Receives the index into the returned page directory
4175	* @param pPdpe Receives the page directory pointer entry. Optional.
4176	*/
4177	DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
4178	{
4179	AssertGCPtr32(GCPtr);
4180
4181	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
4182	AssertReturn(pGuestPDPT, NULL);
4183	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4184	if (pPdpe)
4185	*pPdpe = pGuestPDPT->a[iPdpt];
4186	if (pGuestPDPT->a[iPdpt].n.u1Present)
4187	{
4188	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4189	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4190	PX86PDPAE pGuestPD = NULL;
4191	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
4192	AssertRCReturn(rc, NULL);
4193	#else
4194	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
4195	if ( !pGuestPD
4196	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
4197	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
4198	#endif
4199	*piPD = iPD;
4200	return pGuestPD;
4201	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
4202	}
4203	return NULL;
4204	}
4205
4206	#ifndef IN_RC
4207
4208	/**
4209	* Gets the page map level-4 pointer for the guest.
4210	*
4211	* @returns Pointer to the PML4 page.
4212	* @param pPGM Pointer to the PGM instance data.
4213	*/
4214	DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
4215	{
4216	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4217	PX86PML4 pGuestPml4;
4218	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
4219	AssertRCReturn(rc, NULL);
4220	#else
4221	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
4222	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
4223	if (!pGuestPml4)
4224	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
4225	# endif
4226	Assert(pGuestPml4);
4227	#endif
4228	return pGuestPml4;
4229	}
4230
4231
4232	/**
4233	* Gets the pointer to a page map level-4 entry.
4234	*
4235	* @returns Pointer to the PML4 entry.
4236	* @param pPGM Pointer to the PGM instance data.
4237	* @param iPml4 The index.
4238	*/
4239	DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4240	{
4241	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4242	PX86PML4 pGuestPml4;
4243	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
4244	AssertRCReturn(rc, NULL);
4245	#else
4246	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
4247	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
4248	if (!pGuestPml4)
4249	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
4250	# endif
4251	Assert(pGuestPml4);
4252	#endif
4253	return &pGuestPml4->a[iPml4];
4254	}
4255
4256
4257	/**
4258	* Gets a page map level-4 entry.
4259	*
4260	* @returns The PML4 entry.
4261	* @param pPGM Pointer to the PGM instance data.
4262	* @param iPml4 The index.
4263	*/
4264	DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
4265	{
4266	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4267	PX86PML4 pGuestPml4;
4268	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
4269	if (RT_FAILURE(rc))
4270	{
4271	X86PML4E ZeroPml4e = {0};
4272	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
4273	}
4274	#else
4275	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
4276	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
4277	if (!pGuestPml4)
4278	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
4279	# endif
4280	Assert(pGuestPml4);
4281	#endif
4282	return pGuestPml4->a[iPml4];
4283	}
4284
4285
4286	/**
4287	* Gets the page directory pointer entry for the specified address.
4288	*
4289	* @returns Pointer to the page directory pointer entry in question.
4290	* @returns NULL if the page directory is not present or on an invalid page.
4291	* @param pPGM Pointer to the PGM instance data.
4292	* @param GCPtr The address.
4293	* @param ppPml4e Page Map Level-4 Entry (out)
4294	*/
4295	DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
4296	{
4297	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4298	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4299	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
4300	if (pPml4e->n.u1Present)
4301	{
4302	PX86PDPT pPdpt;
4303	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
4304	AssertRCReturn(rc, NULL);
4305
4306	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4307	return &pPdpt->a[iPdpt];
4308	}
4309	return NULL;
4310	}
4311
4312
4313	/**
4314	* Gets the page directory entry for the specified address.
4315	*
4316	* @returns The page directory entry in question.
4317	* @returns A non-present entry if the page directory is not present or on an invalid page.
4318	* @param pPGM Pointer to the PGM instance data.
4319	* @param GCPtr The address.
4320	* @param ppPml4e Page Map Level-4 Entry (out)
4321	* @param pPdpe Page directory pointer table entry (out)
4322	*/
4323	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
4324	{
4325	X86PDEPAE ZeroPde = {0};
4326	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4327	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4328	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
4329	if (pPml4e->n.u1Present)
4330	{
4331	PCX86PDPT pPdptTemp;
4332	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
4333	AssertRCReturn(rc, ZeroPde);
4334
4335	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4336	*pPdpe = pPdptTemp->a[iPdpt];
4337	if (pPdptTemp->a[iPdpt].n.u1Present)
4338	{
4339	PCX86PDPAE pPD;
4340	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4341	AssertRCReturn(rc, ZeroPde);
4342
4343	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4344	return pPD->a[iPD];
4345	}
4346	}
4347
4348	return ZeroPde;
4349	}
4350
4351
4352	/**
4353	* Gets the page directory entry for the specified address.
4354	*
4355	* @returns The page directory entry in question.
4356	* @returns A non-present entry if the page directory is not present or on an invalid page.
4357	* @param pPGM Pointer to the PGM instance data.
4358	* @param GCPtr The address.
4359	*/
4360	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
4361	{
4362	X86PDEPAE ZeroPde = {0};
4363	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4364	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4365	if (pGuestPml4->a[iPml4].n.u1Present)
4366	{
4367	PCX86PDPT pPdptTemp;
4368	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4369	AssertRCReturn(rc, ZeroPde);
4370
4371	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4372	if (pPdptTemp->a[iPdpt].n.u1Present)
4373	{
4374	PCX86PDPAE pPD;
4375	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4376	AssertRCReturn(rc, ZeroPde);
4377
4378	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4379	return pPD->a[iPD];
4380	}
4381	}
4382	return ZeroPde;
4383	}
4384
4385
4386	/**
4387	* Gets the page directory entry for the specified address.
4388	*
4389	* @returns Pointer to the page directory entry in question.
4390	* @returns NULL if the page directory is not present or on an invalid page.
4391	* @param pPGM Pointer to the PGM instance data.
4392	* @param GCPtr The address.
4393	*/
4394	DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
4395	{
4396	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4397	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4398	if (pGuestPml4->a[iPml4].n.u1Present)
4399	{
4400	PCX86PDPT pPdptTemp;
4401	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4402	AssertRCReturn(rc, NULL);
4403
4404	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4405	if (pPdptTemp->a[iPdpt].n.u1Present)
4406	{
4407	PX86PDPAE pPD;
4408	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4409	AssertRCReturn(rc, NULL);
4410
4411	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4412	return &pPD->a[iPD];
4413	}
4414	}
4415	return NULL;
4416	}
4417
4418
4419	/**
4420	* Gets the GUEST page directory pointer for the specified address.
4421	*
4422	* @returns The page directory in question.
4423	* @returns NULL if the page directory is not present or on an invalid page.
4424	* @param pPGM Pointer to the PGM instance data.
4425	* @param GCPtr The address.
4426	* @param ppPml4e Page Map Level-4 Entry (out)
4427	* @param pPdpe Page directory pointer table entry (out)
4428	* @param piPD Receives the index into the returned page directory
4429	*/
4430	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPE pPdpe, unsigned piPD)
4431	{
4432	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4433	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4434	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
4435	if (pPml4e->n.u1Present)
4436	{
4437	PCX86PDPT pPdptTemp;
4438	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
4439	AssertRCReturn(rc, NULL);
4440
4441	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4442	*pPdpe = pPdptTemp->a[iPdpt];
4443	if (pPdptTemp->a[iPdpt].n.u1Present)
4444	{
4445	PX86PDPAE pPD;
4446	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4447	AssertRCReturn(rc, NULL);
4448
4449	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4450	return pPD;
4451	}
4452	}
4453	return 0;
4454	}
4455
4456	#endif /* !IN_RC */
4457
4458	/**
4459	* Gets the shadow page directory, 32-bit.
4460	*
4461	* @returns Pointer to the shadow 32-bit PD.
4462	* @param pPGM Pointer to the PGM instance data.
4463	*/
4464	DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
4465	{
4466	return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4467	}
4468
4469
4470	/**
4471	* Gets the shadow page directory entry for the specified address, 32-bit.
4472	*
4473	* @returns Shadow 32-bit PDE.
4474	* @param pPGM Pointer to the PGM instance data.
4475	* @param GCPtr The address.
4476	*/
4477	DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4478	{
4479	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4480
4481	PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
4482	if (!pShwPde)
4483	{
4484	X86PDE ZeroPde = {0};
4485	return ZeroPde;
4486	}
4487	return pShwPde->a[iPd];
4488	}
4489
4490
4491	/**
4492	* Gets the pointer to the shadow page directory entry for the specified
4493	* address, 32-bit.
4494	*
4495	* @returns Pointer to the shadow 32-bit PDE.
4496	* @param pPGM Pointer to the PGM instance data.
4497	* @param GCPtr The address.
4498	*/
4499	DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4500	{
4501	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4502
4503	PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
4504	AssertReturn(pPde, NULL);
4505	return &pPde->a[iPd];
4506	}
4507
4508
4509	/**
4510	* Gets the shadow page pointer table, PAE.
4511	*
4512	* @returns Pointer to the shadow PAE PDPT.
4513	* @param pPGM Pointer to the PGM instance data.
4514	*/
4515	DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
4516	{
4517	return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4518	}
4519
4520
4521	/**
4522	* Gets the shadow page directory for the specified address, PAE.
4523	*
4524	* @returns Pointer to the shadow PD.
4525	* @param pPGM Pointer to the PGM instance data.
4526	* @param GCPtr The address.
4527	*/
4528	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4529	{
4530	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4531	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
4532
4533	if (!pPdpt->a[iPdpt].n.u1Present)
4534	return NULL;
4535
4536	/* Fetch the pgm pool shadow descriptor. */
4537	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4538	AssertReturn(pShwPde, NULL);
4539
4540	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4541	}
4542
4543
4544	/**
4545	* Gets the shadow page directory for the specified address, PAE.
4546	*
4547	* @returns Pointer to the shadow PD.
4548	* @param pPGM Pointer to the PGM instance data.
4549	* @param GCPtr The address.
4550	*/
4551	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
4552	{
4553	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4554
4555	if (!pPdpt->a[iPdpt].n.u1Present)
4556	return NULL;
4557
4558	/* Fetch the pgm pool shadow descriptor. */
4559	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4560	AssertReturn(pShwPde, NULL);
4561
4562	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4563	}
4564
4565
4566	/**
4567	* Gets the shadow page directory entry, PAE.
4568	*
4569	* @returns PDE.
4570	* @param pPGM Pointer to the PGM instance data.
4571	* @param GCPtr The address.
4572	*/
4573	DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4574	{
4575	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4576
4577	PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4578	if (!pShwPde)
4579	{
4580	X86PDEPAE ZeroPde = {0};
4581	return ZeroPde;
4582	}
4583	return pShwPde->a[iPd];
4584	}
4585
4586
4587	/**
4588	* Gets the pointer to the shadow page directory entry for an address, PAE.
4589	*
4590	* @returns Pointer to the PDE.
4591	* @param pPGM Pointer to the PGM instance data.
4592	* @param GCPtr The address.
4593	*/
4594	DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4595	{
4596	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4597
4598	PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4599	AssertReturn(pPde, NULL);
4600	return &pPde->a[iPd];
4601	}
4602
4603	#ifndef IN_RC
4604
4605	/**
4606	* Gets the shadow page map level-4 pointer.
4607	*
4608	* @returns Pointer to the shadow PML4.
4609	* @param pPGM Pointer to the PGM instance data.
4610	*/
4611	DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
4612	{
4613	return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4614	}
4615
4616
4617	/**
4618	* Gets the shadow page map level-4 entry for the specified address.
4619	*
4620	* @returns The entry.
4621	* @param pPGM Pointer to the PGM instance data.
4622	* @param GCPtr The address.
4623	*/
4624	DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
4625	{
4626	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4627	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4628
4629	if (!pShwPml4)
4630	{
4631	X86PML4E ZeroPml4e = {0};
4632	return ZeroPml4e;
4633	}
4634	return pShwPml4->a[iPml4];
4635	}
4636
4637
4638	/**
4639	* Gets the pointer to the specified shadow page map level-4 entry.
4640	*
4641	* @returns The entry.
4642	* @param pPGM Pointer to the PGM instance data.
4643	* @param iPml4 The PML4 index.
4644	*/
4645	DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4646	{
4647	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4648	if (!pShwPml4)
4649	return NULL;
4650	return &pShwPml4->a[iPml4];
4651	}
4652
4653
4654	/**
4655	* Gets the GUEST page directory pointer for the specified address.
4656	*
4657	* @returns The page directory in question.
4658	* @returns NULL if the page directory is not present or on an invalid page.
4659	* @param pPGM Pointer to the PGM instance data.
4660	* @param GCPtr The address.
4661	* @param piPD Receives the index into the returned page directory
4662	*/
4663	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
4664	{
4665	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4666	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4667	if (pGuestPml4->a[iPml4].n.u1Present)
4668	{
4669	PCX86PDPT pPdptTemp;
4670	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4671	AssertRCReturn(rc, NULL);
4672
4673	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4674	if (pPdptTemp->a[iPdpt].n.u1Present)
4675	{
4676	PX86PDPAE pPD;
4677	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4678	AssertRCReturn(rc, NULL);
4679
4680	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4681	return pPD;
4682	}
4683	}
4684	return NULL;
4685	}
4686
4687	#endif /* !IN_RC */
4688
4689	/**
4690	* Gets the page state for a physical handler.
4691	*
4692	* @returns The physical handler page state.
4693	* @param pCur The physical handler in question.
4694	*/
4695	DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
4696	{
4697	switch (pCur->enmType)
4698	{
4699	case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
4700	return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
4701
4702	case PGMPHYSHANDLERTYPE_MMIO:
4703	case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
4704	return PGM_PAGE_HNDL_PHYS_STATE_ALL;
4705
4706	default:
4707	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4708	}
4709	}
4710
4711
4712	/**
4713	* Gets the page state for a virtual handler.
4714	*
4715	* @returns The virtual handler page state.
4716	* @param pCur The virtual handler in question.
4717	* @remarks This should never be used on a hypervisor access handler.
4718	*/
4719	DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
4720	{
4721	switch (pCur->enmType)
4722	{
4723	case PGMVIRTHANDLERTYPE_WRITE:
4724	return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
4725	case PGMVIRTHANDLERTYPE_ALL:
4726	return PGM_PAGE_HNDL_VIRT_STATE_ALL;
4727	default:
4728	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4729	}
4730	}
4731
4732
4733	/**
4734	* Clears one physical page of a virtual handler
4735	*
4736	* @param pPGM Pointer to the PGM instance.
4737	* @param pCur Virtual handler structure
4738	* @param iPage Physical page index
4739	*
4740	* @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
4741	* need to care about other handlers in the same page.
4742	*/
4743	DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
4744	{
4745	const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
4746
4747	/*
4748	* Remove the node from the tree (it's supposed to be in the tree if we get here!).
4749	*/
4750	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4751	AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4752	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4753	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4754	#endif
4755	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
4756	{
4757	/* We're the head of the alias chain. */
4758	PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
4759	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4760	AssertReleaseMsg(pRemove != NULL,
4761	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4762	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4763	AssertReleaseMsg(pRemove == pPhys2Virt,
4764	("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4765	" got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4766	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
4767	pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
4768	#endif
4769	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4770	{
4771	/* Insert the next list in the alias chain into the tree. */
4772	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4773	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4774	AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4775	("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4776	pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
4777	#endif
4778	pNext->offNextAlias \|= PGMPHYS2VIRTHANDLER_IS_HEAD;
4779	bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
4780	AssertRelease(fRc);
4781	}
4782	}
4783	else
4784	{
4785	/* Locate the previous node in the alias chain. */
4786	PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
4787	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4788	AssertReleaseMsg(pPrev != pPhys2Virt,
4789	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4790	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4791	#endif
4792	for (;;)
4793	{
4794	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4795	if (pNext == pPhys2Virt)
4796	{
4797	/* unlink. */
4798	LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
4799	pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
4800	if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4801	pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
4802	else
4803	{
4804	PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4805	pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
4806	\| (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
4807	}
4808	break;
4809	}
4810
4811	/* next */
4812	if (pNext == pPrev)
4813	{
4814	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4815	AssertReleaseMsg(pNext != pPrev,
4816	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4817	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4818	#endif
4819	break;
4820	}
4821	pPrev = pNext;
4822	}
4823	}
4824	Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
4825	pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
4826	pPhys2Virt->offNextAlias = 0;
4827	pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
4828
4829	/*
4830	* Clear the ram flags for this page.
4831	*/
4832	PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
4833	AssertReturnVoid(pPage);
4834	PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
4835	}
4836
4837
4838	/**
4839	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4840	*
4841	* @returns Pointer to the shadow page structure.
4842	* @param pPool The pool.
4843	* @param idx The pool page index.
4844	*/
4845	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
4846	{
4847	AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
4848	return &pPool->aPages[idx];
4849	}
4850
4851
4852	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4853	/**
4854	* Clear references to guest physical memory.
4855	*
4856	* @param pPool The pool.
4857	* @param pPoolPage The pool page.
4858	* @param pPhysPage The physical guest page tracking structure.
4859	*/
4860	DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
4861	{
4862	/*
4863	* Just deal with the simple case here.
4864	*/
4865	# ifdef LOG_ENABLED
4866	const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
4867	# endif
4868	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4869	if (cRefs == 1)
4870	{
4871	Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
4872	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
4873	}
4874	else
4875	pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
4876	Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
4877	}
4878	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4879
4880
4881	#ifdef PGMPOOL_WITH_CACHE
4882	/**
4883	* Moves the page to the head of the age list.
4884	*
4885	* This is done when the cached page is used in one way or another.
4886	*
4887	* @param pPool The pool.
4888	* @param pPage The cached page.
4889	*/
4890	DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4891	{
4892	PVM pVM = pPool->CTX_SUFF(pVM);
4893	pgmLock(pVM);
4894
4895	/*
4896	* Move to the head of the age list.
4897	*/
4898	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
4899	{
4900	/* unlink */
4901	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
4902	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
4903	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
4904	else
4905	pPool->iAgeTail = pPage->iAgePrev;
4906
4907	/* insert at head */
4908	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4909	pPage->iAgeNext = pPool->iAgeHead;
4910	Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
4911	pPool->iAgeHead = pPage->idx;
4912	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
4913	}
4914	pgmUnlock(pVM);
4915	}
4916	#endif /* PGMPOOL_WITH_CACHE */
4917
4918	/**
4919	* Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
4920	*
4921	* @param pVM VM Handle.
4922	* @param pPage PGM pool page
4923	*/
4924	DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4925	{
4926	Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
4927	ASMAtomicIncU32(&pPage->cLocked);
4928	}
4929
4930
4931	/**
4932	* Unlocks a page to allow flushing again
4933	*
4934	* @param pVM VM Handle.
4935	* @param pPage PGM pool page
4936	*/
4937	DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4938	{
4939	Assert(PGMIsLockOwner(pPool->CTX_SUFF(pVM)));
4940	Assert(pPage->cLocked);
4941	ASMAtomicDecU32(&pPage->cLocked);
4942	}
4943
4944
4945	/**
4946	* Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
4947	*
4948	* @returns VBox status code.
4949	* @param pPage PGM pool page
4950	*/
4951	DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
4952	{
4953	if (pPage->cLocked)
4954	{
4955	LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
4956	if (pPage->cModifications)
4957	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
4958	return true;
4959	}
4960	return false;
4961	}
4962
4963	/**
4964	* Tells if mappings are to be put into the shadow page table or not
4965	*
4966	* @returns boolean result
4967	* @param pVM VM handle.
4968	*/
4969	DECLINLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
4970	{
4971	#ifdef IN_RING0
4972	/* There are no mappings in VT-x and AMD-V mode. */
4973	Assert(pPGM->fDisableMappings);
4974	return false;
4975	#else
4976	return !pPGM->fDisableMappings;
4977	#endif
4978	}
4979
4980	/** @} */
4981
4982	#endif
4983
4984

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/PGMInternal.h@ 24911

Download in other formats: