PGMAll.cpp@ 17206

Last change on this file since 17206 was 17195, checked in by vboxsync, 16 years ago
VBOX_WITH_PGMPOOL_PAGING_ONLY: deal with hypervisor mappings in guest pae pds that are not yet present
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 76.2 KB

Line
1	/* $Id: PGMAll.cpp 17195 2009-02-27 11:02:41Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	/*******************************************************************************
23	* Header Files *
24	*******************************************************************************/
25	#define LOG_GROUP LOG_GROUP_PGM
26	#include <VBox/pgm.h>
27	#include <VBox/cpum.h>
28	#include <VBox/selm.h>
29	#include <VBox/iom.h>
30	#include <VBox/sup.h>
31	#include <VBox/mm.h>
32	#include <VBox/stam.h>
33	#include <VBox/csam.h>
34	#include <VBox/patm.h>
35	#include <VBox/trpm.h>
36	#include <VBox/rem.h>
37	#include <VBox/em.h>
38	#include <VBox/hwaccm.h>
39	#include <VBox/hwacc_vmx.h>
40	#include "PGMInternal.h"
41	#include <VBox/vm.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/string.h>
45	#include <VBox/log.h>
46	#include <VBox/param.h>
47	#include <VBox/err.h>
48
49
50	/*******************************************************************************
51	* Structures and Typedefs *
52	*******************************************************************************/
53	/**
54	* Stated structure for PGM_GST_NAME(HandlerVirtualUpdate) that's
55	* passed to PGM_GST_NAME(VirtHandlerUpdateOne) during enumeration.
56	*/
57	typedef struct PGMHVUSTATE
58	{
59	/** The VM handle. */
60	PVM pVM;
61	/** The todo flags. */
62	RTUINT fTodo;
63	/** The CR4 register value. */
64	uint32_t cr4;
65	} PGMHVUSTATE, *PPGMHVUSTATE;
66
67
68	/*******************************************************************************
69	* Internal Functions *
70	*******************************************************************************/
71	DECLINLINE(int) pgmShwGetLongModePDPtr(PVM pVM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
72	DECLINLINE(int) pgmShwGetPAEPDPtr(PVM pVM, RTGCPTR GCPtr, PX86PDPT ppPdpt, PX86PDPAE ppPD);
73	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
74	DECLINLINE(int) pgmShwGetPaePoolPagePD(PPGM pPGM, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
75	#endif
76
77	/*
78	* Shadow - 32-bit mode
79	*/
80	#define PGM_SHW_TYPE PGM_TYPE_32BIT
81	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
82	#include "PGMAllShw.h"
83
84	/* Guest - real mode */
85	#define PGM_GST_TYPE PGM_TYPE_REAL
86	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
87	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
88	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
89	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
90	#include "PGMAllGst.h"
91	#include "PGMAllBth.h"
92	#undef BTH_PGMPOOLKIND_PT_FOR_PT
93	#undef BTH_PGMPOOLKIND_ROOT
94	#undef PGM_BTH_NAME
95	#undef PGM_GST_TYPE
96	#undef PGM_GST_NAME
97
98	/* Guest - protected mode */
99	#define PGM_GST_TYPE PGM_TYPE_PROT
100	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
101	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
102	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
103	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
104	#include "PGMAllGst.h"
105	#include "PGMAllBth.h"
106	#undef BTH_PGMPOOLKIND_PT_FOR_PT
107	#undef BTH_PGMPOOLKIND_ROOT
108	#undef PGM_BTH_NAME
109	#undef PGM_GST_TYPE
110	#undef PGM_GST_NAME
111
112	/* Guest - 32-bit mode */
113	#define PGM_GST_TYPE PGM_TYPE_32BIT
114	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
115	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
116	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
117	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
118	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
119	#include "PGMAllGst.h"
120	#include "PGMAllBth.h"
121	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
122	#undef BTH_PGMPOOLKIND_PT_FOR_PT
123	#undef BTH_PGMPOOLKIND_ROOT
124	#undef PGM_BTH_NAME
125	#undef PGM_GST_TYPE
126	#undef PGM_GST_NAME
127
128	#undef PGM_SHW_TYPE
129	#undef PGM_SHW_NAME
130
131
132	/*
133	* Shadow - PAE mode
134	*/
135	#define PGM_SHW_TYPE PGM_TYPE_PAE
136	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
137	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
138	#include "PGMAllShw.h"
139
140	/* Guest - real mode */
141	#define PGM_GST_TYPE PGM_TYPE_REAL
142	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
143	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
144	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
145	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
146	#include "PGMAllBth.h"
147	#undef BTH_PGMPOOLKIND_PT_FOR_PT
148	#undef BTH_PGMPOOLKIND_ROOT
149	#undef PGM_BTH_NAME
150	#undef PGM_GST_TYPE
151	#undef PGM_GST_NAME
152
153	/* Guest - protected mode */
154	#define PGM_GST_TYPE PGM_TYPE_PROT
155	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
156	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
157	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
158	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
159	#include "PGMAllBth.h"
160	#undef BTH_PGMPOOLKIND_PT_FOR_PT
161	#undef BTH_PGMPOOLKIND_ROOT
162	#undef PGM_BTH_NAME
163	#undef PGM_GST_TYPE
164	#undef PGM_GST_NAME
165
166	/* Guest - 32-bit mode */
167	#define PGM_GST_TYPE PGM_TYPE_32BIT
168	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
169	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
170	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
171	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
172	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
173	#include "PGMAllBth.h"
174	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
175	#undef BTH_PGMPOOLKIND_PT_FOR_PT
176	#undef BTH_PGMPOOLKIND_ROOT
177	#undef PGM_BTH_NAME
178	#undef PGM_GST_TYPE
179	#undef PGM_GST_NAME
180
181
182	/* Guest - PAE mode */
183	#define PGM_GST_TYPE PGM_TYPE_PAE
184	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
185	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
186	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
187	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
188	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
189	#include "PGMAllGst.h"
190	#include "PGMAllBth.h"
191	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
192	#undef BTH_PGMPOOLKIND_PT_FOR_PT
193	#undef BTH_PGMPOOLKIND_ROOT
194	#undef PGM_BTH_NAME
195	#undef PGM_GST_TYPE
196	#undef PGM_GST_NAME
197
198	#undef PGM_SHW_TYPE
199	#undef PGM_SHW_NAME
200
201
202	#ifndef IN_RC /* AMD64 implies VT-x/AMD-V */
203	/*
204	* Shadow - AMD64 mode
205	*/
206	# define PGM_SHW_TYPE PGM_TYPE_AMD64
207	# define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
208	# include "PGMAllShw.h"
209
210	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
211	# define PGM_GST_TYPE PGM_TYPE_PROT
212	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
213	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
214	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
215	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
216	# include "PGMAllBth.h"
217	# undef BTH_PGMPOOLKIND_PT_FOR_PT
218	# undef BTH_PGMPOOLKIND_ROOT
219	# undef PGM_BTH_NAME
220	# undef PGM_GST_TYPE
221	# undef PGM_GST_NAME
222
223	# ifdef VBOX_WITH_64_BITS_GUESTS
224	/* Guest - AMD64 mode */
225	# define PGM_GST_TYPE PGM_TYPE_AMD64
226	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
227	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
228	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
229	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
230	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
231	# include "PGMAllGst.h"
232	# include "PGMAllBth.h"
233	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
234	# undef BTH_PGMPOOLKIND_PT_FOR_PT
235	# undef BTH_PGMPOOLKIND_ROOT
236	# undef PGM_BTH_NAME
237	# undef PGM_GST_TYPE
238	# undef PGM_GST_NAME
239	# endif /* VBOX_WITH_64_BITS_GUESTS */
240
241	# undef PGM_SHW_TYPE
242	# undef PGM_SHW_NAME
243
244
245	/*
246	* Shadow - Nested paging mode
247	*/
248	# define PGM_SHW_TYPE PGM_TYPE_NESTED
249	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED(name)
250	# include "PGMAllShw.h"
251
252	/* Guest - real mode */
253	# define PGM_GST_TYPE PGM_TYPE_REAL
254	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
255	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_REAL(name)
256	# include "PGMAllBth.h"
257	# undef PGM_BTH_NAME
258	# undef PGM_GST_TYPE
259	# undef PGM_GST_NAME
260
261	/* Guest - protected mode */
262	# define PGM_GST_TYPE PGM_TYPE_PROT
263	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
264	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PROT(name)
265	# include "PGMAllBth.h"
266	# undef PGM_BTH_NAME
267	# undef PGM_GST_TYPE
268	# undef PGM_GST_NAME
269
270	/* Guest - 32-bit mode */
271	# define PGM_GST_TYPE PGM_TYPE_32BIT
272	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
273	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT(name)
274	# include "PGMAllBth.h"
275	# undef PGM_BTH_NAME
276	# undef PGM_GST_TYPE
277	# undef PGM_GST_NAME
278
279	/* Guest - PAE mode */
280	# define PGM_GST_TYPE PGM_TYPE_PAE
281	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
282	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE(name)
283	# include "PGMAllBth.h"
284	# undef PGM_BTH_NAME
285	# undef PGM_GST_TYPE
286	# undef PGM_GST_NAME
287
288	# ifdef VBOX_WITH_64_BITS_GUESTS
289	/* Guest - AMD64 mode */
290	# define PGM_GST_TYPE PGM_TYPE_AMD64
291	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
292	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64(name)
293	# include "PGMAllBth.h"
294	# undef PGM_BTH_NAME
295	# undef PGM_GST_TYPE
296	# undef PGM_GST_NAME
297	# endif /* VBOX_WITH_64_BITS_GUESTS */
298
299	# undef PGM_SHW_TYPE
300	# undef PGM_SHW_NAME
301
302
303	/*
304	* Shadow - EPT
305	*/
306	# define PGM_SHW_TYPE PGM_TYPE_EPT
307	# define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
308	# include "PGMAllShw.h"
309
310	/* Guest - real mode */
311	# define PGM_GST_TYPE PGM_TYPE_REAL
312	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
313	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
314	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
315	# include "PGMAllBth.h"
316	# undef BTH_PGMPOOLKIND_PT_FOR_PT
317	# undef PGM_BTH_NAME
318	# undef PGM_GST_TYPE
319	# undef PGM_GST_NAME
320
321	/* Guest - protected mode */
322	# define PGM_GST_TYPE PGM_TYPE_PROT
323	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
324	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
325	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
326	# include "PGMAllBth.h"
327	# undef BTH_PGMPOOLKIND_PT_FOR_PT
328	# undef PGM_BTH_NAME
329	# undef PGM_GST_TYPE
330	# undef PGM_GST_NAME
331
332	/* Guest - 32-bit mode */
333	# define PGM_GST_TYPE PGM_TYPE_32BIT
334	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
335	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
336	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
337	# include "PGMAllBth.h"
338	# undef BTH_PGMPOOLKIND_PT_FOR_PT
339	# undef PGM_BTH_NAME
340	# undef PGM_GST_TYPE
341	# undef PGM_GST_NAME
342
343	/* Guest - PAE mode */
344	# define PGM_GST_TYPE PGM_TYPE_PAE
345	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
346	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
347	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
348	# include "PGMAllBth.h"
349	# undef BTH_PGMPOOLKIND_PT_FOR_PT
350	# undef PGM_BTH_NAME
351	# undef PGM_GST_TYPE
352	# undef PGM_GST_NAME
353
354	# ifdef VBOX_WITH_64_BITS_GUESTS
355	/* Guest - AMD64 mode */
356	# define PGM_GST_TYPE PGM_TYPE_AMD64
357	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
358	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
359	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
360	# include "PGMAllBth.h"
361	# undef BTH_PGMPOOLKIND_PT_FOR_PT
362	# undef PGM_BTH_NAME
363	# undef PGM_GST_TYPE
364	# undef PGM_GST_NAME
365	# endif /* VBOX_WITH_64_BITS_GUESTS */
366
367	# undef PGM_SHW_TYPE
368	# undef PGM_SHW_NAME
369
370	#endif /* !IN_RC */
371
372
373	#ifndef IN_RING3
374	/**
375	* #PF Handler.
376	*
377	* @returns VBox status code (appropriate for trap handling and GC return).
378	* @param pVM VM Handle.
379	* @param uErr The trap error code.
380	* @param pRegFrame Trap register frame.
381	* @param pvFault The fault address.
382	*/
383	VMMDECL(int) PGMTrap0eHandler(PVM pVM, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
384	{
385	LogFlow(("PGMTrap0eHandler: uErr=%RGu pvFault=%RGv eip=%RGv\n", uErr, pvFault, (RTGCPTR)pRegFrame->rip));
386	STAM_PROFILE_START(&pVM->pgm.s.StatRZTrap0e, a);
387	STAM_STATS({ pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = NULL; } );
388
389
390	#ifdef VBOX_WITH_STATISTICS
391	/*
392	* Error code stats.
393	*/
394	if (uErr & X86_TRAP_PF_US)
395	{
396	if (!(uErr & X86_TRAP_PF_P))
397	{
398	if (uErr & X86_TRAP_PF_RW)
399	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eUSNotPresentWrite);
400	else
401	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eUSNotPresentRead);
402	}
403	else if (uErr & X86_TRAP_PF_RW)
404	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eUSWrite);
405	else if (uErr & X86_TRAP_PF_RSVD)
406	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eUSReserved);
407	else if (uErr & X86_TRAP_PF_ID)
408	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eUSNXE);
409	else
410	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eUSRead);
411	}
412	else
413	{ /* Supervisor */
414	if (!(uErr & X86_TRAP_PF_P))
415	{
416	if (uErr & X86_TRAP_PF_RW)
417	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eSVNotPresentWrite);
418	else
419	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eSVNotPresentRead);
420	}
421	else if (uErr & X86_TRAP_PF_RW)
422	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eSVWrite);
423	else if (uErr & X86_TRAP_PF_ID)
424	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eSNXE);
425	else if (uErr & X86_TRAP_PF_RSVD)
426	STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eSVReserved);
427	}
428	#endif /* VBOX_WITH_STATISTICS */
429
430	/*
431	* Call the worker.
432	*/
433	int rc = PGM_BTH_PFN(Trap0eHandler, pVM)(pVM, uErr, pRegFrame, pvFault);
434	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
435	rc = VINF_SUCCESS;
436	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVM->pgm.s.StatRZTrap0eGuestPF); });
437	STAM_STATS({ if (!pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution))
438	pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVM->pgm.s.StatRZTrap0eTime2Misc; });
439	STAM_PROFILE_STOP_EX(&pVM->pgm.s.StatRZTrap0e, pVM->pgm.s.CTX_SUFF(pStatTrap0eAttribution), a);
440	return rc;
441	}
442	#endif /* !IN_RING3 */
443
444
445	/**
446	* Prefetch a page
447	*
448	* Typically used to sync commonly used pages before entering raw mode
449	* after a CR3 reload.
450	*
451	* @returns VBox status code suitable for scheduling.
452	* @retval VINF_SUCCESS on success.
453	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
454	* @param pVM VM handle.
455	* @param GCPtrPage Page to invalidate.
456	*/
457	VMMDECL(int) PGMPrefetchPage(PVM pVM, RTGCPTR GCPtrPage)
458	{
459	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,Prefetch), a);
460	int rc = PGM_BTH_PFN(PrefetchPage, pVM)(pVM, GCPtrPage);
461	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,Prefetch), a);
462	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
463	return rc;
464	}
465
466
467	/**
468	* Gets the mapping corresponding to the specified address (if any).
469	*
470	* @returns Pointer to the mapping.
471	* @returns NULL if not
472	*
473	* @param pVM The virtual machine.
474	* @param GCPtr The guest context pointer.
475	*/
476	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr)
477	{
478	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
479	while (pMapping)
480	{
481	if ((uintptr_t)GCPtr < (uintptr_t)pMapping->GCPtr)
482	break;
483	if ((uintptr_t)GCPtr - (uintptr_t)pMapping->GCPtr < pMapping->cb)
484	return pMapping;
485	pMapping = pMapping->CTX_SUFF(pNext);
486	}
487	return NULL;
488	}
489
490
491	/**
492	* Verifies a range of pages for read or write access
493	*
494	* Only checks the guest's page tables
495	*
496	* @returns VBox status code.
497	* @param pVM VM handle.
498	* @param Addr Guest virtual address to check
499	* @param cbSize Access size
500	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
501	* @remarks Current not in use.
502	*/
503	VMMDECL(int) PGMIsValidAccess(PVM pVM, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
504	{
505	/*
506	* Validate input.
507	*/
508	if (fAccess & ~(X86_PTE_US \| X86_PTE_RW))
509	{
510	AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
511	return VERR_INVALID_PARAMETER;
512	}
513
514	uint64_t fPage;
515	int rc = PGMGstGetPage(pVM, (RTGCPTR)Addr, &fPage, NULL);
516	if (RT_FAILURE(rc))
517	{
518	Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
519	return VINF_EM_RAW_GUEST_TRAP;
520	}
521
522	/*
523	* Check if the access would cause a page fault
524	*
525	* Note that hypervisor page directories are not present in the guest's tables, so this check
526	* is sufficient.
527	*/
528	bool fWrite = !!(fAccess & X86_PTE_RW);
529	bool fUser = !!(fAccess & X86_PTE_US);
530	if ( !(fPage & X86_PTE_P)
531	\|\| (fWrite && !(fPage & X86_PTE_RW))
532	\|\| (fUser && !(fPage & X86_PTE_US)) )
533	{
534	Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
535	return VINF_EM_RAW_GUEST_TRAP;
536	}
537	if ( RT_SUCCESS(rc)
538	&& PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
539	return PGMIsValidAccess(pVM, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
540	return rc;
541	}
542
543
544	/**
545	* Verifies a range of pages for read or write access
546	*
547	* Supports handling of pages marked for dirty bit tracking and CSAM
548	*
549	* @returns VBox status code.
550	* @param pVM VM handle.
551	* @param Addr Guest virtual address to check
552	* @param cbSize Access size
553	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
554	*/
555	VMMDECL(int) PGMVerifyAccess(PVM pVM, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
556	{
557	AssertMsg(!(fAccess & ~(X86_PTE_US \| X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
558
559	/*
560	* Get going.
561	*/
562	uint64_t fPageGst;
563	int rc = PGMGstGetPage(pVM, (RTGCPTR)Addr, &fPageGst, NULL);
564	if (RT_FAILURE(rc))
565	{
566	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
567	return VINF_EM_RAW_GUEST_TRAP;
568	}
569
570	/*
571	* Check if the access would cause a page fault
572	*
573	* Note that hypervisor page directories are not present in the guest's tables, so this check
574	* is sufficient.
575	*/
576	const bool fWrite = !!(fAccess & X86_PTE_RW);
577	const bool fUser = !!(fAccess & X86_PTE_US);
578	if ( !(fPageGst & X86_PTE_P)
579	\|\| (fWrite && !(fPageGst & X86_PTE_RW))
580	\|\| (fUser && !(fPageGst & X86_PTE_US)) )
581	{
582	Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
583	return VINF_EM_RAW_GUEST_TRAP;
584	}
585
586	if (!HWACCMIsNestedPagingActive(pVM))
587	{
588	/*
589	* Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
590	*/
591	rc = PGMShwGetPage(pVM, (RTGCPTR)Addr, NULL, NULL);
592	if ( rc == VERR_PAGE_NOT_PRESENT
593	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT)
594	{
595	/*
596	* Page is not present in our page tables.
597	* Try to sync it!
598	*/
599	Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
600	uint32_t uErr = fAccess & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
601	rc = PGM_BTH_PFN(VerifyAccessSyncPage, pVM)(pVM, Addr, fPageGst, uErr);
602	if (rc != VINF_SUCCESS)
603	return rc;
604	}
605	else
606	AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
607	}
608
609	#if 0 /* def VBOX_STRICT; triggers too often now */
610	/*
611	* This check is a bit paranoid, but useful.
612	*/
613	/** @note this will assert when writing to monitored pages (a bit annoying actually) */
614	uint64_t fPageShw;
615	rc = PGMShwGetPage(pVM, (RTGCPTR)Addr, &fPageShw, NULL);
616	if ( (rc == VERR_PAGE_NOT_PRESENT \|\| RT_FAILURE(rc))
617	\|\| (fWrite && !(fPageShw & X86_PTE_RW))
618	\|\| (fUser && !(fPageShw & X86_PTE_US)) )
619	{
620	AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
621	Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
622	return VINF_EM_RAW_GUEST_TRAP;
623	}
624	#endif
625
626	if ( RT_SUCCESS(rc)
627	&& ( PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
628	\|\| Addr + cbSize < Addr))
629	{
630	/* Don't recursively call PGMVerifyAccess as we might run out of stack. */
631	for (;;)
632	{
633	Addr += PAGE_SIZE;
634	if (cbSize > PAGE_SIZE)
635	cbSize -= PAGE_SIZE;
636	else
637	cbSize = 1;
638	rc = PGMVerifyAccess(pVM, Addr, 1, fAccess);
639	if (rc != VINF_SUCCESS)
640	break;
641	if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
642	break;
643	}
644	}
645	return rc;
646	}
647
648
649	/**
650	* Emulation of the invlpg instruction (HC only actually).
651	*
652	* @returns VBox status code, special care required.
653	* @retval VINF_PGM_SYNC_CR3 - handled.
654	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
655	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
656	*
657	* @param pVM VM handle.
658	* @param GCPtrPage Page to invalidate.
659	*
660	* @remark ASSUMES the page table entry or page directory is valid. Fairly
661	* safe, but there could be edge cases!
662	*
663	* @todo Flush page or page directory only if necessary!
664	*/
665	VMMDECL(int) PGMInvalidatePage(PVM pVM, RTGCPTR GCPtrPage)
666	{
667	int rc;
668	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
669
670	#ifndef IN_RING3
671	/*
672	* Notify the recompiler so it can record this instruction.
673	* Failure happens when it's out of space. We'll return to HC in that case.
674	*/
675	rc = REMNotifyInvalidatePage(pVM, GCPtrPage);
676	if (rc != VINF_SUCCESS)
677	return rc;
678	#endif /* !IN_RING3 */
679
680
681	#ifdef IN_RC
682	/*
683	* Check for conflicts and pending CR3 monitoring updates.
684	*/
685	if (!pVM->pgm.s.fMappingsFixed)
686	{
687	if ( pgmGetMapping(pVM, GCPtrPage)
688	&& PGMGstGetPage(pVM, GCPtrPage, NULL, NULL) != VERR_PAGE_TABLE_NOT_PRESENT)
689	{
690	LogFlow(("PGMGCInvalidatePage: Conflict!\n"));
691	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
692	STAM_COUNTER_INC(&pVM->pgm.s.StatRCInvlPgConflict);
693	return VINF_PGM_SYNC_CR3;
694	}
695
696	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
697	{
698	LogFlow(("PGMGCInvalidatePage: PGM_SYNC_MONITOR_CR3 -> reinterpret instruction in R3\n"));
699	STAM_COUNTER_INC(&pVM->pgm.s.StatRCInvlPgSyncMonCR3);
700	return VINF_EM_RAW_EMULATE_INSTR;
701	}
702	}
703	#endif /* IN_RC */
704
705	/*
706	* Call paging mode specific worker.
707	*/
708	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePage), a);
709	rc = PGM_BTH_PFN(InvalidatePage, pVM)(pVM, GCPtrPage);
710	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,InvalidatePage), a);
711
712	#ifdef IN_RING3
713	/*
714	* Check if we have a pending update of the CR3 monitoring.
715	*/
716	if ( RT_SUCCESS(rc)
717	&& (pVM->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3))
718	{
719	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
720	Assert(!pVM->pgm.s.fMappingsFixed);
721	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
722	Assert(pVM->pgm.s.GCPhysCR3 == pVM->pgm.s.GCPhysGstCR3Monitored);
723	rc = PGM_GST_PFN(MonitorCR3, pVM)(pVM, pVM->pgm.s.GCPhysCR3);
724	#endif
725	}
726
727	/*
728	* Inform CSAM about the flush
729	*
730	* Note: This is to check if monitored pages have been changed; when we implement
731	* callbacks for virtual handlers, this is no longer required.
732	*/
733	CSAMR3FlushPage(pVM, GCPtrPage);
734	#endif /* IN_RING3 */
735	return rc;
736	}
737
738
739	/**
740	* Executes an instruction using the interpreter.
741	*
742	* @returns VBox status code (appropriate for trap handling and GC return).
743	* @param pVM VM handle.
744	* @param pRegFrame Register frame.
745	* @param pvFault Fault address.
746	*/
747	VMMDECL(int) PGMInterpretInstruction(PVM pVM, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
748	{
749	uint32_t cb;
750	int rc = EMInterpretInstruction(pVM, pRegFrame, pvFault, &cb);
751	if (rc == VERR_EM_INTERPRETER)
752	rc = VINF_EM_RAW_EMULATE_INSTR;
753	if (rc != VINF_SUCCESS)
754	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", rc, pvFault));
755	return rc;
756	}
757
758
759	/**
760	* Gets effective page information (from the VMM page directory).
761	*
762	* @returns VBox status.
763	* @param pVM VM Handle.
764	* @param GCPtr Guest Context virtual address of the page.
765	* @param pfFlags Where to store the flags. These are X86_PTE_*.
766	* @param pHCPhys Where to store the HC physical address of the page.
767	* This is page aligned.
768	* @remark You should use PGMMapGetPage() for pages in a mapping.
769	*/
770	VMMDECL(int) PGMShwGetPage(PVM pVM, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
771	{
772	return PGM_SHW_PFN(GetPage,pVM)(pVM, GCPtr, pfFlags, pHCPhys);
773	}
774
775
776	/**
777	* Sets (replaces) the page flags for a range of pages in the shadow context.
778	*
779	* @returns VBox status.
780	* @param pVM VM handle.
781	* @param GCPtr The address of the first page.
782	* @param cb The size of the range in bytes.
783	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
784	* @remark You must use PGMMapSetPage() for pages in a mapping.
785	*/
786	VMMDECL(int) PGMShwSetPage(PVM pVM, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
787	{
788	return PGMShwModifyPage(pVM, GCPtr, cb, fFlags, 0);
789	}
790
791
792	/**
793	* Modify page flags for a range of pages in the shadow context.
794	*
795	* The existing flags are ANDed with the fMask and ORed with the fFlags.
796	*
797	* @returns VBox status code.
798	* @param pVM VM handle.
799	* @param GCPtr Virtual address of the first page in the range.
800	* @param cb Size (in bytes) of the range to apply the modification to.
801	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
802	* @param fMask The AND mask - page flags X86_PTE_*.
803	* Be very CAREFUL when ~'ing constants which could be 32-bit!
804	* @remark You must use PGMMapModifyPage() for pages in a mapping.
805	*/
806	VMMDECL(int) PGMShwModifyPage(PVM pVM, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
807	{
808	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
809	Assert(cb);
810
811	/*
812	* Align the input.
813	*/
814	cb += GCPtr & PAGE_OFFSET_MASK;
815	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
816	GCPtr = (GCPtr & PAGE_BASE_GC_MASK); /** @todo this ain't necessary, right... */
817
818	/*
819	* Call worker.
820	*/
821	return PGM_SHW_PFN(ModifyPage, pVM)(pVM, GCPtr, cb, fFlags, fMask);
822	}
823
824
825	/**
826	* Gets the SHADOW page directory pointer for the specified address.
827	*
828	* @returns VBox status.
829	* @param pVM VM handle.
830	* @param GCPtr The address.
831	* @param ppPdpt Receives address of pdpt
832	* @param ppPD Receives address of page directory
833	* @remarks Unused.
834	*/
835	DECLINLINE(int) pgmShwGetPAEPDPtr(PVM pVM, RTGCPTR GCPtr, PX86PDPT ppPdpt, PX86PDPAE ppPD)
836	{
837	PPGM pPGM = &pVM->pgm.s;
838	PPGMPOOL pPool = pPGM->CTX_SUFF(pPool);
839	PPGMPOOLPAGE pShwPage;
840
841	Assert(!HWACCMIsNestedPagingActive(pVM));
842
843	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
844	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(&pVM->pgm.s);
845	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
846
847	*ppPdpt = pPdpt;
848	if (!pPdpe->n.u1Present)
849	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
850
851	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
852	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
853
854	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
855	return VINF_SUCCESS;
856	}
857
858	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
859
860	/**
861	* Gets the shadow page directory for the specified address, PAE.
862	*
863	* @returns Pointer to the shadow PD.
864	* @param pVM VM handle.
865	* @param GCPtr The address.
866	* @param pGstPdpe Guest PDPT entry
867	* @param ppPD Receives address of page directory
868	*/
869	int pgmShwSyncPaePDPtr(PVM pVM, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD)
870	{
871	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
872	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(&pVM->pgm.s);
873	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
874	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
875	PPGMPOOLPAGE pShwPage;
876	int rc;
877
878	/* Allocate page directory if not present. */
879	if ( !pPdpe->n.u1Present
880	&& !(pPdpe->u & X86_PDPE_PG_MASK))
881	{
882	bool fNestedPaging = HWACCMIsNestedPagingActive(pVM);
883	bool fPaging = !!(CPUMGetGuestCR0(pVM) & X86_CR0_PG);
884	RTGCPTR64 GCPdPt;
885	PGMPOOLKIND enmKind;
886
887	# if defined(IN_RC) && defined(VBOX_WITH_PGMPOOL_PAGING_ONLY)
888	/* Make sure the dynamic pPdeDst mapping will not be reused during this function. */
889	PGMDynLockHCPage(pVM, (uint8_t *)pPdpe);
890	# endif
891
892	if (fNestedPaging \|\| !fPaging)
893	{
894	/* AMD-V nested paging or real/protected mode without paging */
895	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
896	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
897	}
898	else
899	{
900	Assert(pGstPdpe);
901
902	if (CPUMGetGuestCR4(pVM) & X86_CR4_PAE)
903	{
904	if (!pGstPdpe->n.u1Present)
905	{
906	/* PD not present; guest must reload CR3 to change it.
907	* No need to monitor anything in this case.
908	*/
909	Assert(!HWACCMIsEnabled(pVM));
910
911	GCPdPt = pGstPdpe->u & X86_PDPE_PG_MASK;
912	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
913	pGstPdpe->n.u1Present = 1;
914	}
915	else
916	{
917	GCPdPt = pGstPdpe->u & X86_PDPE_PG_MASK;
918	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
919	}
920	}
921	else
922	{
923	GCPdPt = CPUMGetGuestCR3(pVM);
924	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
925	}
926	}
927
928	/* Create a reference back to the PDPT by using the index in its shadow page. */
929	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, pVM->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, &pShwPage);
930	if (rc == VERR_PGM_POOL_FLUSHED)
931	{
932	Log(("pgmShwSyncPaePDPtr: PGM pool flushed -> signal sync cr3\n"));
933	Assert(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL);
934	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
935	# if defined(IN_RC) && defined(VBOX_WITH_PGMPOOL_PAGING_ONLY)
936	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdpe);
937	# endif
938	return VINF_PGM_SYNC_CR3;
939	}
940	AssertRCReturn(rc, rc);
941
942	/* The PD was cached or created; hook it up now. */
943	pPdpe->u \|= pShwPage->Core.Key
944	\| (pGstPdpe->u & ~(X86_PDPE_PG_MASK \| X86_PDPE_AVL_MASK \| X86_PDPE_PCD \| X86_PDPE_PWT));
945
946	# if defined(IN_RC) && defined(VBOX_WITH_PGMPOOL_PAGING_ONLY)
947	/* In 32 bits PAE mode we must invalidate the TLB when changing a PDPT entry; the CPU fetches them only during cr3 load, so any
948	* non-present PDPT will continue to cause page faults.
949	*/
950	ASMReloadCR3();
951	PGMDynUnlockHCPage(pVM, (uint8_t *)pPdpe);
952	# endif
953	}
954	else
955	{
956	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
957	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
958
959	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
960	}
961	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
962	return VINF_SUCCESS;
963	}
964
965	/**
966	* Gets the pointer to the shadow page directory entry for an address, PAE.
967	*
968	* @returns Pointer to the PDE.
969	* @param pPGM Pointer to the PGM instance data.
970	* @param GCPtr The address.
971	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
972	*/
973	DECLINLINE(int) pgmShwGetPaePoolPagePD(PPGM pPGM, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
974	{
975	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
976	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
977	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
978	if (!pPdpt->a[iPdPt].n.u1Present)
979	{
980	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, pPdpt->a[iPdPt].u));
981	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
982	}
983
984	/* Fetch the pgm pool shadow descriptor. */
985	PPGMPOOLPAGE pShwPde = pgmPoolGetPageByHCPhys(PGM2VM(pPGM), pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
986	AssertReturn(pShwPde, VERR_INTERNAL_ERROR);
987
988	*ppShwPde = pShwPde;
989	return VINF_SUCCESS;
990	}
991	#endif
992
993	#ifndef IN_RC
994
995	/**
996	* Syncs the SHADOW page directory pointer for the specified address.
997	*
998	* Allocates backing pages in case the PDPT or PML4 entry is missing.
999	*
1000	* The caller is responsible for making sure the guest has a valid PD before
1001	* calling this function.
1002	*
1003	* @returns VBox status.
1004	* @param pVM VM handle.
1005	* @param GCPtr The address.
1006	* @param pGstPml4e Guest PML4 entry
1007	* @param pGstPdpe Guest PDPT entry
1008	* @param ppPD Receives address of page directory
1009	*/
1010	int pgmShwSyncLongModePDPtr(PVM pVM, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD)
1011	{
1012	PPGM pPGM = &pVM->pgm.s;
1013	PPGMPOOL pPool = pPGM->CTX_SUFF(pPool);
1014	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1015	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pPGM, iPml4);
1016	bool fNestedPaging = HWACCMIsNestedPagingActive(pVM);
1017	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1018	bool fPaging = !!(CPUMGetGuestCR0(pVM) & X86_CR0_PG);
1019	#endif
1020	PPGMPOOLPAGE pShwPage;
1021	int rc;
1022
1023	/* Allocate page directory pointer table if not present. */
1024	if ( !pPml4e->n.u1Present
1025	&& !(pPml4e->u & X86_PML4E_PG_MASK))
1026	{
1027	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1028	RTGCPTR64 GCPml4;
1029	PGMPOOLKIND enmKind;
1030
1031	Assert(pVM->pgm.s.CTX_SUFF(pShwPageCR3));
1032
1033	if (fNestedPaging \|\| !fPaging)
1034	{
1035	/* AMD-V nested paging or real/protected mode without paging */
1036	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
1037	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1038	}
1039	else
1040	{
1041	Assert(pGstPml4e && pGstPdpe);
1042
1043	GCPml4 = pGstPml4e->u & X86_PML4E_PG_MASK;
1044	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1045	}
1046
1047	/* Create a reference back to the PDPT by using the index in its shadow page. */
1048	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, pVM->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, &pShwPage);
1049	#else
1050	if (!fNestedPaging)
1051	{
1052	Assert(pGstPml4e && pGstPdpe);
1053	Assert(pVM->pgm.s.CTX_SUFF(pShwPageCR3));
1054
1055	rc = pgmPoolAlloc(pVM, pGstPml4e->u & X86_PML4E_PG_MASK,
1056	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT, pVM->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, &pShwPage);
1057	}
1058	else
1059	{
1060	/* AMD-V nested paging. (Intel EPT never comes here) */
1061	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1062	rc = pgmPoolAlloc(pVM, GCPml4 + RT_BIT_64(63) /* hack: make the address unique */,
1063	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS, PGMPOOL_IDX_NESTED_ROOT, iPml4, &pShwPage);
1064	}
1065	#endif
1066	if (rc == VERR_PGM_POOL_FLUSHED)
1067	{
1068	Log(("PGMShwSyncLongModePDPtr: PGM pool flushed (1) -> signal sync cr3\n"));
1069	Assert(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL);
1070	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
1071	return VINF_PGM_SYNC_CR3;
1072	}
1073	AssertRCReturn(rc, rc);
1074	}
1075	else
1076	{
1077	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1078	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
1079	}
1080	/* The PDPT was cached or created; hook it up now. */
1081	pPml4e->u \|= pShwPage->Core.Key
1082	\| (pGstPml4e->u & ~(X86_PML4E_PG_MASK \| X86_PML4E_AVL_MASK \| X86_PML4E_PCD \| X86_PML4E_PWT));
1083
1084	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1085	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1086	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1087
1088	/* Allocate page directory if not present. */
1089	if ( !pPdpe->n.u1Present
1090	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1091	{
1092	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1093	RTGCPTR64 GCPdPt;
1094	PGMPOOLKIND enmKind;
1095
1096	if (fNestedPaging \|\| !fPaging)
1097	{
1098	/* AMD-V nested paging or real/protected mode without paging */
1099	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1100	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1101	}
1102	else
1103	{
1104	Assert(pGstPdpe);
1105
1106	GCPdPt = pGstPdpe->u & X86_PDPE_PG_MASK;
1107	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1108	}
1109
1110	/* Create a reference back to the PDPT by using the index in its shadow page. */
1111	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, pShwPage->idx, iPdPt, &pShwPage);
1112	#else
1113	if (!fNestedPaging)
1114	{
1115	Assert(pGstPml4e && pGstPdpe);
1116	Assert(!(pPdpe->u & X86_PDPE_PG_MASK));
1117	/* Create a reference back to the PDPT by using the index in its shadow page. */
1118	rc = pgmPoolAlloc(pVM, pGstPdpe->u & X86_PDPE_PG_MASK, PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD, pShwPage->idx, iPdPt, &pShwPage);
1119	}
1120	else
1121	{
1122	/* AMD-V nested paging. (Intel EPT never comes here) */
1123	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1124
1125	rc = pgmPoolAlloc(pVM, GCPdPt + RT_BIT_64(62) /* hack: make the address unique */, PGMPOOLKIND_64BIT_PD_FOR_PHYS, pShwPage->idx, iPdPt, &pShwPage);
1126	}
1127	#endif
1128	if (rc == VERR_PGM_POOL_FLUSHED)
1129	{
1130	Log(("PGMShwSyncLongModePDPtr: PGM pool flushed (2) -> signal sync cr3\n"));
1131	Assert(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL);
1132	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
1133	return VINF_PGM_SYNC_CR3;
1134	}
1135	AssertRCReturn(rc, rc);
1136	}
1137	else
1138	{
1139	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1140	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
1141	}
1142	/* The PD was cached or created; hook it up now. */
1143	pPdpe->u \|= pShwPage->Core.Key
1144	\| (pGstPdpe->u & ~(X86_PDPE_PG_MASK \| X86_PDPE_AVL_MASK \| X86_PDPE_PCD \| X86_PDPE_PWT));
1145
1146	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1147	return VINF_SUCCESS;
1148	}
1149
1150
1151	/**
1152	* Gets the SHADOW page directory pointer for the specified address (long mode).
1153	*
1154	* @returns VBox status.
1155	* @param pVM VM handle.
1156	* @param GCPtr The address.
1157	* @param ppPdpt Receives address of pdpt
1158	* @param ppPD Receives address of page directory
1159	*/
1160	DECLINLINE(int) pgmShwGetLongModePDPtr(PVM pVM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1161	{
1162	PPGM pPGM = &pVM->pgm.s;
1163	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1164	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pPGM, iPml4);
1165	AssertReturn(pPml4e, VERR_INTERNAL_ERROR);
1166	if (ppPml4e)
1167	*ppPml4e = (PX86PML4E)pPml4e;
1168	if (!pPml4e->n.u1Present)
1169	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1170
1171	PPGMPOOL pPool = pPGM->CTX_SUFF(pPool);
1172	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1173	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
1174
1175	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1176	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1177	if (!pPdpt->a[iPdPt].n.u1Present)
1178	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1179
1180	pShwPage = pgmPoolGetPage(pPool, pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1181	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
1182
1183	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1184	return VINF_SUCCESS;
1185	}
1186
1187
1188	/**
1189	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1190	* backing pages in case the PDPT or PML4 entry is missing.
1191	*
1192	* @returns VBox status.
1193	* @param pVM VM handle.
1194	* @param GCPtr The address.
1195	* @param ppPdpt Receives address of pdpt
1196	* @param ppPD Receives address of page directory
1197	*/
1198	int pgmShwGetEPTPDPtr(PVM pVM, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1199	{
1200	PPGM pPGM = &pVM->pgm.s;
1201	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1202	PPGMPOOL pPool = pPGM->CTX_SUFF(pPool);
1203	PEPTPML4 pPml4;
1204	PEPTPML4E pPml4e;
1205	PPGMPOOLPAGE pShwPage;
1206	int rc;
1207
1208	Assert(HWACCMIsNestedPagingActive(pVM));
1209
1210	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1211	rc = PGM_HCPHYS_2_PTR(pVM, pPGM->HCPhysShwNestedRoot, &pPml4);
1212	AssertRCReturn(rc, rc);
1213	# else
1214	pPml4 = (PEPTPML4)pPGM->CTX_SUFF(pShwNestedRoot);
1215	# endif
1216	Assert(pPml4);
1217
1218	/* Allocate page directory pointer table if not present. */
1219	pPml4e = &pPml4->a[iPml4];
1220	if ( !pPml4e->n.u1Present
1221	&& !(pPml4e->u & EPT_PML4E_PG_MASK))
1222	{
1223	Assert(!(pPml4e->u & EPT_PML4E_PG_MASK));
1224	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1225
1226	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1227	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOL_IDX_NESTED_ROOT, iPml4, &pShwPage);
1228	#else
1229	rc = pgmPoolAlloc(pVM, GCPml4 + RT_BIT_64(63) /* hack: make the address unique */, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOL_IDX_NESTED_ROOT, iPml4, &pShwPage);
1230	#endif
1231	if (rc == VERR_PGM_POOL_FLUSHED)
1232	{
1233	Log(("PGMShwSyncEPTPDPtr: PGM pool flushed (1) -> signal sync cr3\n"));
1234	Assert(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL);
1235	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
1236	return VINF_PGM_SYNC_CR3;
1237	}
1238	AssertRCReturn(rc, rc);
1239	}
1240	else
1241	{
1242	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1243	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
1244	}
1245	/* The PDPT was cached or created; hook it up now and fill with the default value. */
1246	pPml4e->u = pShwPage->Core.Key;
1247	pPml4e->n.u1Present = 1;
1248	pPml4e->n.u1Write = 1;
1249	pPml4e->n.u1Execute = 1;
1250
1251	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1252	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1253	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1254
1255	if (ppPdpt)
1256	*ppPdpt = pPdpt;
1257
1258	/* Allocate page directory if not present. */
1259	if ( !pPdpe->n.u1Present
1260	&& !(pPdpe->u & EPT_PDPTE_PG_MASK))
1261	{
1262	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1263
1264	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1265	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_64BIT_PD_FOR_PHYS, pShwPage->idx, iPdPt, &pShwPage);
1266	#else
1267	rc = pgmPoolAlloc(pVM, GCPdPt + RT_BIT_64(62) /* hack: make the address unique */, PGMPOOLKIND_64BIT_PD_FOR_PHYS, pShwPage->idx, iPdPt, &pShwPage);
1268	#endif
1269	if (rc == VERR_PGM_POOL_FLUSHED)
1270	{
1271	Log(("PGMShwSyncEPTPDPtr: PGM pool flushed (2) -> signal sync cr3\n"));
1272	Assert(pVM->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL);
1273	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
1274	return VINF_PGM_SYNC_CR3;
1275	}
1276	AssertRCReturn(rc, rc);
1277	}
1278	else
1279	{
1280	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1281	AssertReturn(pShwPage, VERR_INTERNAL_ERROR);
1282	}
1283	/* The PD was cached or created; hook it up now and fill with the default value. */
1284	pPdpe->u = pShwPage->Core.Key;
1285	pPdpe->n.u1Present = 1;
1286	pPdpe->n.u1Write = 1;
1287	pPdpe->n.u1Execute = 1;
1288
1289	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pShwPage);
1290	return VINF_SUCCESS;
1291	}
1292
1293	#endif /* IN_RC */
1294
1295	/**
1296	* Gets effective Guest OS page information.
1297	*
1298	* When GCPtr is in a big page, the function will return as if it was a normal
1299	* 4KB page. If the need for distinguishing between big and normal page becomes
1300	* necessary at a later point, a PGMGstGetPage() will be created for that
1301	* purpose.
1302	*
1303	* @returns VBox status.
1304	* @param pVM VM Handle.
1305	* @param GCPtr Guest Context virtual address of the page.
1306	* @param pfFlags Where to store the flags. These are X86_PTE_*, even for big pages.
1307	* @param pGCPhys Where to store the GC physical address of the page.
1308	* This is page aligned. The fact that the
1309	*/
1310	VMMDECL(int) PGMGstGetPage(PVM pVM, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
1311	{
1312	return PGM_GST_PFN(GetPage,pVM)(pVM, GCPtr, pfFlags, pGCPhys);
1313	}
1314
1315
1316	/**
1317	* Checks if the page is present.
1318	*
1319	* @returns true if the page is present.
1320	* @returns false if the page is not present.
1321	* @param pVM The VM handle.
1322	* @param GCPtr Address within the page.
1323	*/
1324	VMMDECL(bool) PGMGstIsPagePresent(PVM pVM, RTGCPTR GCPtr)
1325	{
1326	int rc = PGMGstGetPage(pVM, GCPtr, NULL, NULL);
1327	return RT_SUCCESS(rc);
1328	}
1329
1330
1331	/**
1332	* Sets (replaces) the page flags for a range of pages in the guest's tables.
1333	*
1334	* @returns VBox status.
1335	* @param pVM VM handle.
1336	* @param GCPtr The address of the first page.
1337	* @param cb The size of the range in bytes.
1338	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
1339	*/
1340	VMMDECL(int) PGMGstSetPage(PVM pVM, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
1341	{
1342	return PGMGstModifyPage(pVM, GCPtr, cb, fFlags, 0);
1343	}
1344
1345
1346	/**
1347	* Modify page flags for a range of pages in the guest's tables
1348	*
1349	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1350	*
1351	* @returns VBox status code.
1352	* @param pVM VM handle.
1353	* @param GCPtr Virtual address of the first page in the range.
1354	* @param cb Size (in bytes) of the range to apply the modification to.
1355	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1356	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
1357	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1358	*/
1359	VMMDECL(int) PGMGstModifyPage(PVM pVM, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
1360	{
1361	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,GstModifyPage), a);
1362
1363	/*
1364	* Validate input.
1365	*/
1366	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1367	Assert(cb);
1368
1369	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
1370
1371	/*
1372	* Adjust input.
1373	*/
1374	cb += GCPtr & PAGE_OFFSET_MASK;
1375	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
1376	GCPtr = (GCPtr & PAGE_BASE_GC_MASK);
1377
1378	/*
1379	* Call worker.
1380	*/
1381	int rc = PGM_GST_PFN(ModifyPage, pVM)(pVM, GCPtr, cb, fFlags, fMask);
1382
1383	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,GstModifyPage), a);
1384	return rc;
1385	}
1386
1387
1388	/**
1389	* Gets the specified page directory pointer table entry.
1390	*
1391	* @returns PDP entry
1392	* @param pPGM Pointer to the PGM instance data.
1393	* @param iPdpt PDPT index
1394	*/
1395	VMMDECL(X86PDPE) PGMGstGetPaePDPtr(PVM pVM, unsigned iPdpt)
1396	{
1397	Assert(iPdpt <= 3);
1398	return pgmGstGetPaePDPTPtr(&pVM->pgm.s)->a[iPdpt & 3];
1399	}
1400
1401
1402	/**
1403	* Gets the current CR3 register value for the shadow memory context.
1404	* @returns CR3 value.
1405	* @param pVM The VM handle.
1406	*/
1407	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVM pVM)
1408	{
1409	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1410	PGMMODE enmShadowMode = pVM->pgm.s.enmShadowMode;
1411	switch (enmShadowMode)
1412	{
1413	case PGMMODE_EPT:
1414	return pVM->pgm.s.HCPhysShwNestedRoot;
1415
1416	default:
1417	return pVM->pgm.s.HCPhysShwCR3;
1418	}
1419	#else
1420	PGMMODE enmShadowMode = pVM->pgm.s.enmShadowMode;
1421	switch (enmShadowMode)
1422	{
1423	case PGMMODE_32_BIT:
1424	return pVM->pgm.s.HCPhysShw32BitPD;
1425
1426	case PGMMODE_PAE:
1427	case PGMMODE_PAE_NX:
1428	return pVM->pgm.s.HCPhysShwPaePdpt;
1429
1430	case PGMMODE_AMD64:
1431	case PGMMODE_AMD64_NX:
1432	return pVM->pgm.s.HCPhysShwCR3;
1433
1434	case PGMMODE_EPT:
1435	return pVM->pgm.s.HCPhysShwNestedRoot;
1436
1437	case PGMMODE_NESTED:
1438	return PGMGetNestedCR3(pVM, PGMGetHostMode(pVM));
1439
1440	default:
1441	AssertMsgFailed(("enmShadowMode=%d\n", enmShadowMode));
1442	return ~0;
1443	}
1444	#endif
1445	}
1446
1447
1448	/**
1449	* Gets the current CR3 register value for the nested memory context.
1450	* @returns CR3 value.
1451	* @param pVM The VM handle.
1452	*/
1453	VMMDECL(RTHCPHYS) PGMGetNestedCR3(PVM pVM, PGMMODE enmShadowMode)
1454	{
1455	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1456	return pVM->pgm.s.HCPhysShwCR3;
1457	#else
1458	switch (enmShadowMode)
1459	{
1460	case PGMMODE_32_BIT:
1461	return pVM->pgm.s.HCPhysShw32BitPD;
1462
1463	case PGMMODE_PAE:
1464	case PGMMODE_PAE_NX:
1465	return pVM->pgm.s.HCPhysShwPaePdpt;
1466
1467	case PGMMODE_AMD64:
1468	case PGMMODE_AMD64_NX:
1469	return pVM->pgm.s.HCPhysShwCR3;
1470
1471	default:
1472	AssertMsgFailed(("enmShadowMode=%d\n", enmShadowMode));
1473	return ~0;
1474	}
1475	#endif
1476	}
1477
1478
1479	/**
1480	* Gets the current CR3 register value for the EPT paging memory context.
1481	* @returns CR3 value.
1482	* @param pVM The VM handle.
1483	*/
1484	VMMDECL(RTHCPHYS) PGMGetEPTCR3(PVM pVM)
1485	{
1486	return pVM->pgm.s.HCPhysShwNestedRoot;
1487	}
1488
1489
1490	/**
1491	* Gets the CR3 register value for the 32-Bit shadow memory context.
1492	* @returns CR3 value.
1493	* @param pVM The VM handle.
1494	*/
1495	VMMDECL(RTHCPHYS) PGMGetHyper32BitCR3(PVM pVM)
1496	{
1497	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1498	return pVM->pgm.s.HCPhysShwCR3;
1499	#else
1500	return pVM->pgm.s.HCPhysShw32BitPD;
1501	#endif
1502	}
1503
1504
1505	/**
1506	* Gets the CR3 register value for the PAE shadow memory context.
1507	* @returns CR3 value.
1508	* @param pVM The VM handle.
1509	*/
1510	VMMDECL(RTHCPHYS) PGMGetHyperPaeCR3(PVM pVM)
1511	{
1512	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1513	return pVM->pgm.s.HCPhysShwCR3;
1514	#else
1515	return pVM->pgm.s.HCPhysShwPaePdpt;
1516	#endif
1517	}
1518
1519
1520	/**
1521	* Gets the CR3 register value for the AMD64 shadow memory context.
1522	* @returns CR3 value.
1523	* @param pVM The VM handle.
1524	*/
1525	VMMDECL(RTHCPHYS) PGMGetHyperAmd64CR3(PVM pVM)
1526	{
1527	return pVM->pgm.s.HCPhysShwCR3;
1528	}
1529
1530	/**
1531	* Gets the current CR3 register value for the HC intermediate memory context.
1532	* @returns CR3 value.
1533	* @param pVM The VM handle.
1534	*/
1535	VMMDECL(RTHCPHYS) PGMGetInterHCCR3(PVM pVM)
1536	{
1537	switch (pVM->pgm.s.enmHostMode)
1538	{
1539	case SUPPAGINGMODE_32_BIT:
1540	case SUPPAGINGMODE_32_BIT_GLOBAL:
1541	return pVM->pgm.s.HCPhysInterPD;
1542
1543	case SUPPAGINGMODE_PAE:
1544	case SUPPAGINGMODE_PAE_GLOBAL:
1545	case SUPPAGINGMODE_PAE_NX:
1546	case SUPPAGINGMODE_PAE_GLOBAL_NX:
1547	return pVM->pgm.s.HCPhysInterPaePDPT;
1548
1549	case SUPPAGINGMODE_AMD64:
1550	case SUPPAGINGMODE_AMD64_GLOBAL:
1551	case SUPPAGINGMODE_AMD64_NX:
1552	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
1553	return pVM->pgm.s.HCPhysInterPaePDPT;
1554
1555	default:
1556	AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode));
1557	return ~0;
1558	}
1559	}
1560
1561
1562	/**
1563	* Gets the current CR3 register value for the RC intermediate memory context.
1564	* @returns CR3 value.
1565	* @param pVM The VM handle.
1566	*/
1567	VMMDECL(RTHCPHYS) PGMGetInterRCCR3(PVM pVM)
1568	{
1569	switch (pVM->pgm.s.enmShadowMode)
1570	{
1571	case PGMMODE_32_BIT:
1572	return pVM->pgm.s.HCPhysInterPD;
1573
1574	case PGMMODE_PAE:
1575	case PGMMODE_PAE_NX:
1576	return pVM->pgm.s.HCPhysInterPaePDPT;
1577
1578	case PGMMODE_AMD64:
1579	case PGMMODE_AMD64_NX:
1580	return pVM->pgm.s.HCPhysInterPaePML4;
1581
1582	case PGMMODE_EPT:
1583	case PGMMODE_NESTED:
1584	return 0; /* not relevant */
1585
1586	default:
1587	AssertMsgFailed(("enmShadowMode=%d\n", pVM->pgm.s.enmShadowMode));
1588	return ~0;
1589	}
1590	}
1591
1592
1593	/**
1594	* Gets the CR3 register value for the 32-Bit intermediate memory context.
1595	* @returns CR3 value.
1596	* @param pVM The VM handle.
1597	*/
1598	VMMDECL(RTHCPHYS) PGMGetInter32BitCR3(PVM pVM)
1599	{
1600	return pVM->pgm.s.HCPhysInterPD;
1601	}
1602
1603
1604	/**
1605	* Gets the CR3 register value for the PAE intermediate memory context.
1606	* @returns CR3 value.
1607	* @param pVM The VM handle.
1608	*/
1609	VMMDECL(RTHCPHYS) PGMGetInterPaeCR3(PVM pVM)
1610	{
1611	return pVM->pgm.s.HCPhysInterPaePDPT;
1612	}
1613
1614
1615	/**
1616	* Gets the CR3 register value for the AMD64 intermediate memory context.
1617	* @returns CR3 value.
1618	* @param pVM The VM handle.
1619	*/
1620	VMMDECL(RTHCPHYS) PGMGetInterAmd64CR3(PVM pVM)
1621	{
1622	return pVM->pgm.s.HCPhysInterPaePML4;
1623	}
1624
1625
1626	/**
1627	* Performs and schedules necessary updates following a CR3 load or reload.
1628	*
1629	* This will normally involve mapping the guest PD or nPDPT
1630	*
1631	* @returns VBox status code.
1632	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
1633	* safely be ignored and overridden since the FF will be set too then.
1634	* @param pVM VM handle.
1635	* @param cr3 The new cr3.
1636	* @param fGlobal Indicates whether this is a global flush or not.
1637	*/
1638	VMMDECL(int) PGMFlushTLB(PVM pVM, uint64_t cr3, bool fGlobal)
1639	{
1640	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,FlushTLB), a);
1641
1642	/*
1643	* Always flag the necessary updates; necessary for hardware acceleration
1644	*/
1645	/** @todo optimize this, it shouldn't always be necessary. */
1646	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL);
1647	if (fGlobal)
1648	VM_FF_SET(pVM, VM_FF_PGM_SYNC_CR3);
1649	LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVM->pgm.s.GCPhysCR3, fGlobal));
1650
1651	/*
1652	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
1653	*/
1654	int rc = VINF_SUCCESS;
1655	RTGCPHYS GCPhysCR3;
1656	if ( pVM->pgm.s.enmGuestMode == PGMMODE_PAE
1657	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_PAE_NX
1658	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_AMD64
1659	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_AMD64_NX)
1660	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
1661	else
1662	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
1663	if (pVM->pgm.s.GCPhysCR3 != GCPhysCR3)
1664	{
1665	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1666	/* Unmap the old CR3 value before activating the new one. */
1667	rc = PGM_BTH_PFN(UnmapCR3, pVM)(pVM);
1668	AssertRC(rc);
1669	#endif
1670	RTGCPHYS GCPhysOldCR3 = pVM->pgm.s.GCPhysCR3;
1671	pVM->pgm.s.GCPhysCR3 = GCPhysCR3;
1672	rc = PGM_BTH_PFN(MapCR3, pVM)(pVM, GCPhysCR3);
1673	if (RT_LIKELY(rc == VINF_SUCCESS))
1674	{
1675	if (!pVM->pgm.s.fMappingsFixed)
1676	{
1677	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1678	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
1679	rc = PGM_GST_PFN(MonitorCR3, pVM)(pVM, GCPhysCR3);
1680	#endif
1681	}
1682	}
1683	else
1684	{
1685	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
1686	Assert(VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL \| VM_FF_PGM_SYNC_CR3));
1687	pVM->pgm.s.GCPhysCR3 = GCPhysOldCR3;
1688	pVM->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
1689	if (!pVM->pgm.s.fMappingsFixed)
1690	pVM->pgm.s.fSyncFlags \|= PGM_SYNC_MONITOR_CR3;
1691	}
1692
1693	if (fGlobal)
1694	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
1695	else
1696	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,FlushTLBNewCR3));
1697	}
1698	else
1699	{
1700	/*
1701	* Check if we have a pending update of the CR3 monitoring.
1702	*/
1703	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
1704	{
1705	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1706	Assert(!pVM->pgm.s.fMappingsFixed);
1707	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
1708	rc = PGM_GST_PFN(MonitorCR3, pVM)(pVM, GCPhysCR3);
1709	#endif
1710	}
1711	if (fGlobal)
1712	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
1713	else
1714	STAM_COUNTER_INC(&pVM->pgm.s.CTX_MID_Z(Stat,FlushTLBSameCR3));
1715	}
1716
1717	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,FlushTLB), a);
1718	return rc;
1719	}
1720
1721
1722	/**
1723	* Performs and schedules necessary updates following a CR3 load or reload when
1724	* using nested or extended paging.
1725	*
1726	* This API is an alterantive to PDMFlushTLB that avoids actually flushing the
1727	* TLB and triggering a SyncCR3.
1728	*
1729	* This will normally involve mapping the guest PD or nPDPT
1730	*
1731	* @returns VBox status code.
1732	* @retval VINF_SUCCESS.
1733	* @retval (If applied when not in nested mode: VINF_PGM_SYNC_CR3 if monitoring
1734	* requires a CR3 sync. This can safely be ignored and overridden since
1735	* the FF will be set too then.)
1736	* @param pVM VM handle.
1737	* @param cr3 The new cr3.
1738	*/
1739	VMMDECL(int) PGMUpdateCR3(PVM pVM, uint64_t cr3)
1740	{
1741	LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVM->pgm.s.GCPhysCR3));
1742
1743	/* We assume we're only called in nested paging mode. */
1744	Assert(pVM->pgm.s.fMappingsFixed);
1745	Assert(!(pVM->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3));
1746	Assert(pVM->pgm.s.enmShadowMode == PGMMODE_NESTED \|\| pVM->pgm.s.enmShadowMode == PGMMODE_EPT);
1747
1748	/*
1749	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
1750	*/
1751	int rc = VINF_SUCCESS;
1752	RTGCPHYS GCPhysCR3;
1753	if ( pVM->pgm.s.enmGuestMode == PGMMODE_PAE
1754	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_PAE_NX
1755	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_AMD64
1756	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_AMD64_NX)
1757	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
1758	else
1759	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
1760	if (pVM->pgm.s.GCPhysCR3 != GCPhysCR3)
1761	{
1762	pVM->pgm.s.GCPhysCR3 = GCPhysCR3;
1763	rc = PGM_BTH_PFN(MapCR3, pVM)(pVM, GCPhysCR3);
1764	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. */
1765	}
1766	return rc;
1767	}
1768
1769
1770	/**
1771	* Synchronize the paging structures.
1772	*
1773	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
1774	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
1775	* in several places, most importantly whenever the CR3 is loaded.
1776	*
1777	* @returns VBox status code.
1778	* @param pVM The virtual machine.
1779	* @param cr0 Guest context CR0 register
1780	* @param cr3 Guest context CR3 register
1781	* @param cr4 Guest context CR4 register
1782	* @param fGlobal Including global page directories or not
1783	*/
1784	VMMDECL(int) PGMSyncCR3(PVM pVM, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
1785	{
1786	int rc;
1787
1788	/*
1789	* We might be called when we shouldn't.
1790	*
1791	* The mode switching will ensure that the PD is resynced
1792	* after every mode switch. So, if we find ourselves here
1793	* when in protected or real mode we can safely disable the
1794	* FF and return immediately.
1795	*/
1796	if (pVM->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
1797	{
1798	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
1799	VM_FF_CLEAR(pVM, VM_FF_PGM_SYNC_CR3);
1800	VM_FF_CLEAR(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL);
1801	return VINF_SUCCESS;
1802	}
1803
1804	/* If global pages are not supported, then all flushes are global. */
1805	if (!(cr4 & X86_CR4_PGE))
1806	fGlobal = true;
1807	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
1808	VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL)));
1809
1810	#ifdef PGMPOOL_WITH_MONITORING
1811	/*
1812	* The pool may have pending stuff and even require a return to ring-3 to
1813	* clear the whole thing.
1814	*/
1815	rc = pgmPoolSyncCR3(pVM);
1816	if (rc != VINF_SUCCESS)
1817	return rc;
1818	#endif
1819
1820	/*
1821	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
1822	* This should be done before SyncCR3.
1823	*/
1824	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
1825	{
1826	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
1827
1828	RTGCPHYS GCPhysCR3Old = pVM->pgm.s.GCPhysCR3;
1829	RTGCPHYS GCPhysCR3;
1830	if ( pVM->pgm.s.enmGuestMode == PGMMODE_PAE
1831	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_PAE_NX
1832	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_AMD64
1833	\|\| pVM->pgm.s.enmGuestMode == PGMMODE_AMD64_NX)
1834	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
1835	else
1836	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
1837
1838	#ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
1839	if (pVM->pgm.s.GCPhysCR3 != GCPhysCR3)
1840	{
1841	/* Unmap the old CR3 value before activating the new one. */
1842	rc = PGM_BTH_PFN(UnmapCR3, pVM)(pVM);
1843	AssertRC(rc);
1844	}
1845	#endif
1846
1847	pVM->pgm.s.GCPhysCR3 = GCPhysCR3;
1848	rc = PGM_BTH_PFN(MapCR3, pVM)(pVM, GCPhysCR3);
1849	#ifdef IN_RING3
1850	if (rc == VINF_PGM_SYNC_CR3)
1851	rc = pgmPoolSyncCR3(pVM);
1852	#else
1853	if (rc == VINF_PGM_SYNC_CR3)
1854	{
1855	pVM->pgm.s.GCPhysCR3 = GCPhysCR3Old;
1856	return rc;
1857	}
1858	#endif
1859	AssertRCReturn(rc, rc);
1860	AssertRCSuccessReturn(rc, VERR_INTERNAL_ERROR);
1861	}
1862
1863	/*
1864	* Let the 'Bth' function do the work and we'll just keep track of the flags.
1865	*/
1866	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3), a);
1867	rc = PGM_BTH_PFN(SyncCR3, pVM)(pVM, cr0, cr3, cr4, fGlobal);
1868	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3), a);
1869	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1870	if (rc == VINF_SUCCESS)
1871	{
1872	if (!(pVM->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
1873	{
1874	VM_FF_CLEAR(pVM, VM_FF_PGM_SYNC_CR3);
1875	VM_FF_CLEAR(pVM, VM_FF_PGM_SYNC_CR3_NON_GLOBAL);
1876	}
1877
1878	/*
1879	* Check if we have a pending update of the CR3 monitoring.
1880	*/
1881	if (pVM->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
1882	{
1883	pVM->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1884	Assert(!pVM->pgm.s.fMappingsFixed);
1885	#ifndef VBOX_WITH_PGMPOOL_PAGING_ONLY
1886	Assert(pVM->pgm.s.GCPhysCR3 == pVM->pgm.s.GCPhysGstCR3Monitored);
1887	rc = PGM_GST_PFN(MonitorCR3, pVM)(pVM, pVM->pgm.s.GCPhysCR3);
1888	#endif
1889	}
1890	}
1891
1892	/*
1893	* Now flush the CR3 (guest context).
1894	*/
1895	if (rc == VINF_SUCCESS)
1896	PGM_INVL_GUEST_TLBS();
1897	return rc;
1898	}
1899
1900
1901	/**
1902	* Called whenever CR0 or CR4 in a way which may change
1903	* the paging mode.
1904	*
1905	* @returns VBox status code fit for scheduling in GC and R0.
1906	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
1907	* @retval VINF_PGM_CHANGE_MODE if we're in GC or R0 and the mode changes.
1908	* @param pVM VM handle.
1909	* @param cr0 The new cr0.
1910	* @param cr4 The new cr4.
1911	* @param efer The new extended feature enable register.
1912	*/
1913	VMMDECL(int) PGMChangeMode(PVM pVM, uint64_t cr0, uint64_t cr4, uint64_t efer)
1914	{
1915	PGMMODE enmGuestMode;
1916
1917	/*
1918	* Calc the new guest mode.
1919	*/
1920	if (!(cr0 & X86_CR0_PE))
1921	enmGuestMode = PGMMODE_REAL;
1922	else if (!(cr0 & X86_CR0_PG))
1923	enmGuestMode = PGMMODE_PROTECTED;
1924	else if (!(cr4 & X86_CR4_PAE))
1925	enmGuestMode = PGMMODE_32_BIT;
1926	else if (!(efer & MSR_K6_EFER_LME))
1927	{
1928	if (!(efer & MSR_K6_EFER_NXE))
1929	enmGuestMode = PGMMODE_PAE;
1930	else
1931	enmGuestMode = PGMMODE_PAE_NX;
1932	}
1933	else
1934	{
1935	if (!(efer & MSR_K6_EFER_NXE))
1936	enmGuestMode = PGMMODE_AMD64;
1937	else
1938	enmGuestMode = PGMMODE_AMD64_NX;
1939	}
1940
1941	/*
1942	* Did it change?
1943	*/
1944	if (pVM->pgm.s.enmGuestMode == enmGuestMode)
1945	return VINF_SUCCESS;
1946
1947	/* Flush the TLB */
1948	PGM_INVL_GUEST_TLBS();
1949
1950	#ifdef IN_RING3
1951	return PGMR3ChangeMode(pVM, enmGuestMode);
1952	#else
1953	LogFlow(("PGMChangeMode: returns VINF_PGM_CHANGE_MODE.\n"));
1954	return VINF_PGM_CHANGE_MODE;
1955	#endif
1956	}
1957
1958
1959	/**
1960	* Gets the current guest paging mode.
1961	*
1962	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
1963	*
1964	* @returns The current paging mode.
1965	* @param pVM The VM handle.
1966	*/
1967	VMMDECL(PGMMODE) PGMGetGuestMode(PVM pVM)
1968	{
1969	return pVM->pgm.s.enmGuestMode;
1970	}
1971
1972
1973	/**
1974	* Gets the current shadow paging mode.
1975	*
1976	* @returns The current paging mode.
1977	* @param pVM The VM handle.
1978	*/
1979	VMMDECL(PGMMODE) PGMGetShadowMode(PVM pVM)
1980	{
1981	return pVM->pgm.s.enmShadowMode;
1982	}
1983
1984	/**
1985	* Gets the current host paging mode.
1986	*
1987	* @returns The current paging mode.
1988	* @param pVM The VM handle.
1989	*/
1990	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
1991	{
1992	switch (pVM->pgm.s.enmHostMode)
1993	{
1994	case SUPPAGINGMODE_32_BIT:
1995	case SUPPAGINGMODE_32_BIT_GLOBAL:
1996	return PGMMODE_32_BIT;
1997
1998	case SUPPAGINGMODE_PAE:
1999	case SUPPAGINGMODE_PAE_GLOBAL:
2000	return PGMMODE_PAE;
2001
2002	case SUPPAGINGMODE_PAE_NX:
2003	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2004	return PGMMODE_PAE_NX;
2005
2006	case SUPPAGINGMODE_AMD64:
2007	case SUPPAGINGMODE_AMD64_GLOBAL:
2008	return PGMMODE_AMD64;
2009
2010	case SUPPAGINGMODE_AMD64_NX:
2011	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2012	return PGMMODE_AMD64_NX;
2013
2014	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
2015	}
2016
2017	return PGMMODE_INVALID;
2018	}
2019
2020
2021	/**
2022	* Get mode name.
2023	*
2024	* @returns read-only name string.
2025	* @param enmMode The mode which name is desired.
2026	*/
2027	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
2028	{
2029	switch (enmMode)
2030	{
2031	case PGMMODE_REAL: return "Real";
2032	case PGMMODE_PROTECTED: return "Protected";
2033	case PGMMODE_32_BIT: return "32-bit";
2034	case PGMMODE_PAE: return "PAE";
2035	case PGMMODE_PAE_NX: return "PAE+NX";
2036	case PGMMODE_AMD64: return "AMD64";
2037	case PGMMODE_AMD64_NX: return "AMD64+NX";
2038	case PGMMODE_NESTED: return "Nested";
2039	case PGMMODE_EPT: return "EPT";
2040	default: return "unknown mode value";
2041	}
2042	}
2043
2044
2045	/**
2046	* Acquire the PGM lock.
2047	*
2048	* @returns VBox status code
2049	* @param pVM The VM to operate on.
2050	*/
2051	int pgmLock(PVM pVM)
2052	{
2053	int rc = PDMCritSectEnter(&pVM->pgm.s.CritSect, VERR_SEM_BUSY);
2054	#ifdef IN_RC
2055	if (rc == VERR_SEM_BUSY)
2056	rc = VMMGCCallHost(pVM, VMMCALLHOST_PGM_LOCK, 0);
2057	#elif defined(IN_RING0)
2058	if (rc == VERR_SEM_BUSY)
2059	rc = VMMR0CallHost(pVM, VMMCALLHOST_PGM_LOCK, 0);
2060	#endif
2061	AssertRC(rc);
2062	return rc;
2063	}
2064
2065
2066	/**
2067	* Release the PGM lock.
2068	*
2069	* @returns VBox status code
2070	* @param pVM The VM to operate on.
2071	*/
2072	void pgmUnlock(PVM pVM)
2073	{
2074	PDMCritSectLeave(&pVM->pgm.s.CritSect);
2075	}
2076
2077	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2078
2079	/**
2080	* Temporarily maps one guest page specified by GC physical address.
2081	* These pages must have a physical mapping in HC, i.e. they cannot be MMIO pages.
2082	*
2083	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
2084	* reused after 8 mappings (or perhaps a few more if you score with the cache).
2085	*
2086	* @returns VBox status.
2087	* @param pVM VM handle.
2088	* @param GCPhys GC Physical address of the page.
2089	* @param ppv Where to store the address of the mapping.
2090	*/
2091	VMMDECL(int) PGMDynMapGCPage(PVM pVM, RTGCPHYS GCPhys, void **ppv)
2092	{
2093	AssertMsg(!(GCPhys & PAGE_OFFSET_MASK), ("GCPhys=%RGp\n", GCPhys));
2094
2095	/*
2096	* Get the ram range.
2097	*/
2098	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2099	while (pRam && GCPhys - pRam->GCPhys >= pRam->cb)
2100	pRam = pRam->CTX_SUFF(pNext);
2101	if (!pRam)
2102	{
2103	AssertMsgFailed(("Invalid physical address %RGp!\n", GCPhys));
2104	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
2105	}
2106
2107	/*
2108	* Pass it on to PGMDynMapHCPage.
2109	*/
2110	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT]);
2111	//Log(("PGMDynMapGCPage: GCPhys=%RGp HCPhys=%RHp\n", GCPhys, HCPhys));
2112	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2113	pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
2114	#else
2115	PGMDynMapHCPage(pVM, HCPhys, ppv);
2116	#endif
2117	return VINF_SUCCESS;
2118	}
2119
2120
2121	/**
2122	* Temporarily maps one guest page specified by unaligned GC physical address.
2123	* These pages must have a physical mapping in HC, i.e. they cannot be MMIO pages.
2124	*
2125	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
2126	* reused after 8 mappings (or perhaps a few more if you score with the cache).
2127	*
2128	* The caller is aware that only the speicifed page is mapped and that really bad things
2129	* will happen if writing beyond the page!
2130	*
2131	* @returns VBox status.
2132	* @param pVM VM handle.
2133	* @param GCPhys GC Physical address within the page to be mapped.
2134	* @param ppv Where to store the address of the mapping address corresponding to GCPhys.
2135	*/
2136	VMMDECL(int) PGMDynMapGCPageOff(PVM pVM, RTGCPHYS GCPhys, void **ppv)
2137	{
2138	/*
2139	* Get the ram range.
2140	*/
2141	PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
2142	while (pRam && GCPhys - pRam->GCPhys >= pRam->cb)
2143	pRam = pRam->CTX_SUFF(pNext);
2144	if (!pRam)
2145	{
2146	AssertMsgFailed(("Invalid physical address %RGp!\n", GCPhys));
2147	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
2148	}
2149
2150	/*
2151	* Pass it on to PGMDynMapHCPage.
2152	*/
2153	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT]);
2154	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2155	pgmR0DynMapHCPageInlined(&pVM->pgm.s, HCPhys, ppv);
2156	#else
2157	PGMDynMapHCPage(pVM, HCPhys, ppv);
2158	#endif
2159	ppv = (void )((uintptr_t)*ppv \| (GCPhys & PAGE_OFFSET_MASK));
2160	return VINF_SUCCESS;
2161	}
2162
2163
2164	# ifdef IN_RC
2165	/**
2166	* Temporarily maps one host page specified by HC physical address.
2167	*
2168	* Be WARNED that the dynamic page mapping area is small, 16 pages, thus the space is
2169	* reused after 16 mappings (or perhaps a few more if you score with the cache).
2170	*
2171	* @returns VINF_SUCCESS, will bail out to ring-3 on failure.
2172	* @param pVM VM handle.
2173	* @param HCPhys HC Physical address of the page.
2174	* @param ppv Where to store the address of the mapping. This is the
2175	* address of the PAGE not the exact address corresponding
2176	* to HCPhys. Use PGMDynMapHCPageOff if you care for the
2177	* page offset.
2178	*/
2179	VMMDECL(int) PGMDynMapHCPage(PVM pVM, RTHCPHYS HCPhys, void **ppv)
2180	{
2181	AssertMsg(!(HCPhys & PAGE_OFFSET_MASK), ("HCPhys=%RHp\n", HCPhys));
2182
2183	/*
2184	* Check the cache.
2185	*/
2186	register unsigned iCache;
2187	for (iCache = 0;iCache < RT_ELEMENTS(pVM->pgm.s.aHCPhysDynPageMapCache);iCache++)
2188	{
2189	static const uint8_t au8Trans[MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT][RT_ELEMENTS(pVM->pgm.s.aHCPhysDynPageMapCache)] =
2190	{
2191	{ 0, 9, 10, 11, 12, 13, 14, 15},
2192	{ 0, 1, 10, 11, 12, 13, 14, 15},
2193	{ 0, 1, 2, 11, 12, 13, 14, 15},
2194	{ 0, 1, 2, 3, 12, 13, 14, 15},
2195	{ 0, 1, 2, 3, 4, 13, 14, 15},
2196	{ 0, 1, 2, 3, 4, 5, 14, 15},
2197	{ 0, 1, 2, 3, 4, 5, 6, 15},
2198	{ 0, 1, 2, 3, 4, 5, 6, 7},
2199	{ 8, 1, 2, 3, 4, 5, 6, 7},
2200	{ 8, 9, 2, 3, 4, 5, 6, 7},
2201	{ 8, 9, 10, 3, 4, 5, 6, 7},
2202	{ 8, 9, 10, 11, 4, 5, 6, 7},
2203	{ 8, 9, 10, 11, 12, 5, 6, 7},
2204	{ 8, 9, 10, 11, 12, 13, 6, 7},
2205	{ 8, 9, 10, 11, 12, 13, 14, 7},
2206	{ 8, 9, 10, 11, 12, 13, 14, 15},
2207	};
2208	AssertCompile(RT_ELEMENTS(au8Trans) == 16);
2209	AssertCompile(RT_ELEMENTS(au8Trans[0]) == 8);
2210
2211	if (pVM->pgm.s.aHCPhysDynPageMapCache[iCache] == HCPhys)
2212	{
2213	int iPage = au8Trans[pVM->pgm.s.iDynPageMapLast][iCache];
2214	void *pv = pVM->pgm.s.pbDynPageMapBaseGC + (iPage << PAGE_SHIFT);
2215	*ppv = pv;
2216	STAM_COUNTER_INC(&pVM->pgm.s.StatRCDynMapCacheHits);
2217	Log4(("PGMGCDynMapHCPage: HCPhys=%RHp pv=%p iPage=%d iCache=%d\n", HCPhys, pv, iPage, iCache));
2218	return VINF_SUCCESS;
2219	}
2220	}
2221	AssertCompile(RT_ELEMENTS(pVM->pgm.s.aHCPhysDynPageMapCache) == 8);
2222	AssertCompile((MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT) == 16);
2223	STAM_COUNTER_INC(&pVM->pgm.s.StatRCDynMapCacheMisses);
2224
2225	/*
2226	* Update the page tables.
2227	*/
2228	register unsigned iPage = pVM->pgm.s.iDynPageMapLast;
2229	# ifdef VBOX_WITH_PGMPOOL_PAGING_ONLY
2230	unsigned i;
2231	for (i=0;i<(MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT);i++)
2232	{
2233	pVM->pgm.s.iDynPageMapLast = iPage = (iPage + 1) & ((MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT) - 1);
2234	if (!(pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage].u & PGM_PTFLAGS_DYN_LOCKED))
2235	break;
2236	iPage++;
2237	}
2238	AssertRelease(i != (MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT));
2239	# else
2240	pVM->pgm.s.iDynPageMapLast = iPage = (iPage + 1) & ((MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT) - 1);
2241	# endif
2242
2243	pVM->pgm.s.aHCPhysDynPageMapCache[iPage & (RT_ELEMENTS(pVM->pgm.s.aHCPhysDynPageMapCache) - 1)] = HCPhys;
2244	pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage].u = (uint32_t)HCPhys \| X86_PTE_P \| X86_PTE_A \| X86_PTE_D;
2245	pVM->pgm.s.paDynPageMapPaePTEsGC[iPage].u = HCPhys \| X86_PTE_P \| X86_PTE_A \| X86_PTE_D;
2246
2247	void *pv = pVM->pgm.s.pbDynPageMapBaseGC + (iPage << PAGE_SHIFT);
2248	*ppv = pv;
2249	ASMInvalidatePage(pv);
2250	Log4(("PGMGCDynMapHCPage: HCPhys=%RHp pv=%p iPage=%d\n", HCPhys, pv, iPage));
2251	return VINF_SUCCESS;
2252	}
2253
2254	/**
2255	* Temporarily lock a dynamic page to prevent it from being reused.
2256	*
2257	* @returns VINF_SUCCESS, will bail out to ring-3 on failure.
2258	* @param pVM VM handle.
2259	* @param GCPage GC address of page
2260	*/
2261	VMMDECL(int) PGMDynLockHCPage(PVM pVM, RCPTRTYPE(uint8_t *) GCPage)
2262	{
2263	unsigned iPage;
2264
2265	Assert(GCPage >= pVM->pgm.s.pbDynPageMapBaseGC && GCPage < (pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE));
2266	iPage = ((uintptr_t)(GCPage - pVM->pgm.s.pbDynPageMapBaseGC)) >> PAGE_SHIFT;
2267	Assert(!(pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage].u & PGM_PTFLAGS_DYN_LOCKED));
2268	pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage].u \|= PGM_PTFLAGS_DYN_LOCKED;
2269	return VINF_SUCCESS;
2270	}
2271
2272	/**
2273	* Unlock a dynamic page
2274	*
2275	* @returns VINF_SUCCESS, will bail out to ring-3 on failure.
2276	* @param pVM VM handle.
2277	* @param GCPage GC address of page
2278	*/
2279	VMMDECL(int) PGMDynUnlockHCPage(PVM pVM, RCPTRTYPE(uint8_t *) GCPage)
2280	{
2281	unsigned iPage;
2282
2283	Assert(GCPage >= pVM->pgm.s.pbDynPageMapBaseGC && GCPage < (pVM->pgm.s.pbDynPageMapBaseGC + MM_HYPER_DYNAMIC_SIZE));
2284	iPage = ((uintptr_t)(GCPage - pVM->pgm.s.pbDynPageMapBaseGC)) >> PAGE_SHIFT;
2285	Assert(pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage].u & PGM_PTFLAGS_DYN_LOCKED);
2286	pVM->pgm.s.paDynPageMap32BitPTEsGC[iPage].u &= ~PGM_PTFLAGS_DYN_LOCKED;
2287	return VINF_SUCCESS;
2288	}
2289
2290	# ifdef VBOX_STRICT
2291	/**
2292	* Check for lock leaks.
2293	*
2294	* @param pVM VM handle.
2295	*/
2296	VMMDECL(void) PGMDynCheckLocks(PVM pVM)
2297	{
2298	for (unsigned i=0;i<(MM_HYPER_DYNAMIC_SIZE >> PAGE_SHIFT);i++)
2299	Assert(!(pVM->pgm.s.paDynPageMap32BitPTEsGC[i].u & PGM_PTFLAGS_DYN_LOCKED));
2300	}
2301	# endif
2302	# endif /* IN_RC */
2303
2304	#endif /* IN_RC \|\| VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
2305	#ifdef VBOX_STRICT
2306
2307	/**
2308	* Asserts that there are no mapping conflicts.
2309	*
2310	* @returns Number of conflicts.
2311	* @param pVM The VM Handle.
2312	*/
2313	VMMDECL(unsigned) PGMAssertNoMappingConflicts(PVM pVM)
2314	{
2315	unsigned cErrors = 0;
2316
2317	/*
2318	* Check for mapping conflicts.
2319	*/
2320	for (PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
2321	pMapping;
2322	pMapping = pMapping->CTX_SUFF(pNext))
2323	{
2324	/** @todo This is slow and should be optimized, but since it's just assertions I don't care now. */
2325	for (RTGCPTR GCPtr = pMapping->GCPtr;
2326	GCPtr <= pMapping->GCPtrLast;
2327	GCPtr += PAGE_SIZE)
2328	{
2329	int rc = PGMGstGetPage(pVM, (RTGCPTR)GCPtr, NULL, NULL);
2330	if (rc != VERR_PAGE_TABLE_NOT_PRESENT)
2331	{
2332	AssertMsgFailed(("Conflict at %RGv with %s\n", GCPtr, R3STRING(pMapping->pszDesc)));
2333	cErrors++;
2334	break;
2335	}
2336	}
2337	}
2338
2339	return cErrors;
2340	}
2341
2342
2343	/**
2344	* Asserts that everything related to the guest CR3 is correctly shadowed.
2345	*
2346	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
2347	* and assert the correctness of the guest CR3 mapping before asserting that the
2348	* shadow page tables is in sync with the guest page tables.
2349	*
2350	* @returns Number of conflicts.
2351	* @param pVM The VM Handle.
2352	* @param cr3 The current guest CR3 register value.
2353	* @param cr4 The current guest CR4 register value.
2354	*/
2355	VMMDECL(unsigned) PGMAssertCR3(PVM pVM, uint64_t cr3, uint64_t cr4)
2356	{
2357	STAM_PROFILE_START(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3), a);
2358	unsigned cErrors = PGM_BTH_PFN(AssertCR3, pVM)(pVM, cr3, cr4, 0, ~(RTGCPTR)0);
2359	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_MID_Z(Stat,SyncCR3), a);
2360	return cErrors;
2361	}
2362
2363	#endif /* VBOX_STRICT */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 17206

Download in other formats: