PGMAll.cpp@ 73249

Last change on this file since 73249 was 73249, checked in by vboxsync, 6 years ago
PGM: Working on eliminating PGMMODEDATA and the corresponding PGMCPU section so we can do mode switching in ring-0. This third part starts dealing with shadow+guest paging pointers. bugref:9044
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 118.1 KB

Line
1	/* $Id: PGMAll.cpp 73249 2018-07-19 17:29:09Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM
23	#include <VBox/vmm/pgm.h>
24	#include <VBox/vmm/cpum.h>
25	#include <VBox/vmm/selm.h>
26	#include <VBox/vmm/iem.h>
27	#include <VBox/vmm/iom.h>
28	#include <VBox/sup.h>
29	#include <VBox/vmm/mm.h>
30	#include <VBox/vmm/stam.h>
31	#include <VBox/vmm/csam.h>
32	#include <VBox/vmm/patm.h>
33	#include <VBox/vmm/trpm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#include <VBox/vmm/em.h>
38	#include <VBox/vmm/hm.h>
39	#include <VBox/vmm/hm_vmx.h>
40	#include "PGMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include "PGMInline.h"
43	#include <iprt/assert.h>
44	#include <iprt/asm-amd64-x86.h>
45	#include <iprt/string.h>
46	#include <VBox/log.h>
47	#include <VBox/param.h>
48	#include <VBox/err.h>
49
50
51	/*********************************************************************************************************************************
52	* Structures and Typedefs *
53	*********************************************************************************************************************************/
54	/**
55	* Stated structure for PGM_GST_NAME(HandlerVirtualUpdate) that's
56	* passed to PGM_GST_NAME(VirtHandlerUpdateOne) during enumeration.
57	*/
58	typedef struct PGMHVUSTATE
59	{
60	/** Pointer to the VM. */
61	PVM pVM;
62	/** Pointer to the VMCPU. */
63	PVMCPU pVCpu;
64	/** The todo flags. */
65	RTUINT fTodo;
66	/** The CR4 register value. */
67	uint32_t cr4;
68	} PGMHVUSTATE, *PPGMHVUSTATE;
69
70
71	/*********************************************************************************************************************************
72	* Internal Functions *
73	*********************************************************************************************************************************/
74	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
75	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPU pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
76	#ifndef IN_RC
77	static int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
78	static int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
79	#endif
80
81
82	/*
83	* Shadow - 32-bit mode
84	*/
85	#define PGM_SHW_TYPE PGM_TYPE_32BIT
86	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
87	#include "PGMAllShw.h"
88
89	/* Guest - real mode */
90	#define PGM_GST_TYPE PGM_TYPE_REAL
91	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
92	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
93	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
94	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
95	#include "PGMGstDefs.h"
96	#include "PGMAllGst.h"
97	#include "PGMAllBth.h"
98	#undef BTH_PGMPOOLKIND_PT_FOR_PT
99	#undef BTH_PGMPOOLKIND_ROOT
100	#undef PGM_BTH_NAME
101	#undef PGM_GST_TYPE
102	#undef PGM_GST_NAME
103
104	/* Guest - protected mode */
105	#define PGM_GST_TYPE PGM_TYPE_PROT
106	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
107	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
108	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
109	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
110	#include "PGMGstDefs.h"
111	#include "PGMAllGst.h"
112	#include "PGMAllBth.h"
113	#undef BTH_PGMPOOLKIND_PT_FOR_PT
114	#undef BTH_PGMPOOLKIND_ROOT
115	#undef PGM_BTH_NAME
116	#undef PGM_GST_TYPE
117	#undef PGM_GST_NAME
118
119	/* Guest - 32-bit mode */
120	#define PGM_GST_TYPE PGM_TYPE_32BIT
121	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
122	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
123	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
124	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
125	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
126	#include "PGMGstDefs.h"
127	#include "PGMAllGst.h"
128	#include "PGMAllBth.h"
129	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
130	#undef BTH_PGMPOOLKIND_PT_FOR_PT
131	#undef BTH_PGMPOOLKIND_ROOT
132	#undef PGM_BTH_NAME
133	#undef PGM_GST_TYPE
134	#undef PGM_GST_NAME
135
136	#undef PGM_SHW_TYPE
137	#undef PGM_SHW_NAME
138
139
140	/*
141	* Shadow - PAE mode
142	*/
143	#define PGM_SHW_TYPE PGM_TYPE_PAE
144	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
145	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
146	#include "PGMAllShw.h"
147
148	/* Guest - real mode */
149	#define PGM_GST_TYPE PGM_TYPE_REAL
150	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
151	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
152	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
153	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
154	#include "PGMGstDefs.h"
155	#include "PGMAllBth.h"
156	#undef BTH_PGMPOOLKIND_PT_FOR_PT
157	#undef BTH_PGMPOOLKIND_ROOT
158	#undef PGM_BTH_NAME
159	#undef PGM_GST_TYPE
160	#undef PGM_GST_NAME
161
162	/* Guest - protected mode */
163	#define PGM_GST_TYPE PGM_TYPE_PROT
164	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
165	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
166	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
167	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
168	#include "PGMGstDefs.h"
169	#include "PGMAllBth.h"
170	#undef BTH_PGMPOOLKIND_PT_FOR_PT
171	#undef BTH_PGMPOOLKIND_ROOT
172	#undef PGM_BTH_NAME
173	#undef PGM_GST_TYPE
174	#undef PGM_GST_NAME
175
176	/* Guest - 32-bit mode */
177	#define PGM_GST_TYPE PGM_TYPE_32BIT
178	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
179	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
180	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
181	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
182	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
183	#include "PGMGstDefs.h"
184	#include "PGMAllBth.h"
185	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
186	#undef BTH_PGMPOOLKIND_PT_FOR_PT
187	#undef BTH_PGMPOOLKIND_ROOT
188	#undef PGM_BTH_NAME
189	#undef PGM_GST_TYPE
190	#undef PGM_GST_NAME
191
192
193	/* Guest - PAE mode */
194	#define PGM_GST_TYPE PGM_TYPE_PAE
195	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
196	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
197	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
198	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
199	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
200	#include "PGMGstDefs.h"
201	#include "PGMAllGst.h"
202	#include "PGMAllBth.h"
203	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
204	#undef BTH_PGMPOOLKIND_PT_FOR_PT
205	#undef BTH_PGMPOOLKIND_ROOT
206	#undef PGM_BTH_NAME
207	#undef PGM_GST_TYPE
208	#undef PGM_GST_NAME
209
210	#undef PGM_SHW_TYPE
211	#undef PGM_SHW_NAME
212
213
214	#ifndef IN_RC /* AMD64 implies VT-x/AMD-V */
215	/*
216	* Shadow - AMD64 mode
217	*/
218	# define PGM_SHW_TYPE PGM_TYPE_AMD64
219	# define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
220	# include "PGMAllShw.h"
221
222	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
223	/** @todo retire this hack. */
224	# define PGM_GST_TYPE PGM_TYPE_PROT
225	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
226	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
227	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
228	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
229	# include "PGMGstDefs.h"
230	# include "PGMAllBth.h"
231	# undef BTH_PGMPOOLKIND_PT_FOR_PT
232	# undef BTH_PGMPOOLKIND_ROOT
233	# undef PGM_BTH_NAME
234	# undef PGM_GST_TYPE
235	# undef PGM_GST_NAME
236
237	# ifdef VBOX_WITH_64_BITS_GUESTS
238	/* Guest - AMD64 mode */
239	# define PGM_GST_TYPE PGM_TYPE_AMD64
240	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
241	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
242	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
243	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
244	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
245	# include "PGMGstDefs.h"
246	# include "PGMAllGst.h"
247	# include "PGMAllBth.h"
248	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
249	# undef BTH_PGMPOOLKIND_PT_FOR_PT
250	# undef BTH_PGMPOOLKIND_ROOT
251	# undef PGM_BTH_NAME
252	# undef PGM_GST_TYPE
253	# undef PGM_GST_NAME
254	# endif /* VBOX_WITH_64_BITS_GUESTS */
255
256	# undef PGM_SHW_TYPE
257	# undef PGM_SHW_NAME
258
259
260	/*
261	* Shadow - 32-bit nested paging mode.
262	*/
263	# define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
264	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
265	# include "PGMAllShw.h"
266
267	/* Guest - real mode */
268	# define PGM_GST_TYPE PGM_TYPE_REAL
269	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
270	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
271	# include "PGMGstDefs.h"
272	# include "PGMAllBth.h"
273	# undef PGM_BTH_NAME
274	# undef PGM_GST_TYPE
275	# undef PGM_GST_NAME
276
277	/* Guest - protected mode */
278	# define PGM_GST_TYPE PGM_TYPE_PROT
279	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
280	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
281	# include "PGMGstDefs.h"
282	# include "PGMAllBth.h"
283	# undef PGM_BTH_NAME
284	# undef PGM_GST_TYPE
285	# undef PGM_GST_NAME
286
287	/* Guest - 32-bit mode */
288	# define PGM_GST_TYPE PGM_TYPE_32BIT
289	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
290	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
291	# include "PGMGstDefs.h"
292	# include "PGMAllBth.h"
293	# undef PGM_BTH_NAME
294	# undef PGM_GST_TYPE
295	# undef PGM_GST_NAME
296
297	/* Guest - PAE mode */
298	# define PGM_GST_TYPE PGM_TYPE_PAE
299	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
300	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
301	# include "PGMGstDefs.h"
302	# include "PGMAllBth.h"
303	# undef PGM_BTH_NAME
304	# undef PGM_GST_TYPE
305	# undef PGM_GST_NAME
306
307	# ifdef VBOX_WITH_64_BITS_GUESTS
308	/* Guest - AMD64 mode */
309	# define PGM_GST_TYPE PGM_TYPE_AMD64
310	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
311	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
312	# include "PGMGstDefs.h"
313	# include "PGMAllBth.h"
314	# undef PGM_BTH_NAME
315	# undef PGM_GST_TYPE
316	# undef PGM_GST_NAME
317	# endif /* VBOX_WITH_64_BITS_GUESTS */
318
319	# undef PGM_SHW_TYPE
320	# undef PGM_SHW_NAME
321
322
323	/*
324	* Shadow - PAE nested paging mode.
325	*/
326	# define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
327	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
328	# include "PGMAllShw.h"
329
330	/* Guest - real mode */
331	# define PGM_GST_TYPE PGM_TYPE_REAL
332	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
333	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
334	# include "PGMGstDefs.h"
335	# include "PGMAllBth.h"
336	# undef PGM_BTH_NAME
337	# undef PGM_GST_TYPE
338	# undef PGM_GST_NAME
339
340	/* Guest - protected mode */
341	# define PGM_GST_TYPE PGM_TYPE_PROT
342	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
343	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
344	# include "PGMGstDefs.h"
345	# include "PGMAllBth.h"
346	# undef PGM_BTH_NAME
347	# undef PGM_GST_TYPE
348	# undef PGM_GST_NAME
349
350	/* Guest - 32-bit mode */
351	# define PGM_GST_TYPE PGM_TYPE_32BIT
352	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
353	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
354	# include "PGMGstDefs.h"
355	# include "PGMAllBth.h"
356	# undef PGM_BTH_NAME
357	# undef PGM_GST_TYPE
358	# undef PGM_GST_NAME
359
360	/* Guest - PAE mode */
361	# define PGM_GST_TYPE PGM_TYPE_PAE
362	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
363	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
364	# include "PGMGstDefs.h"
365	# include "PGMAllBth.h"
366	# undef PGM_BTH_NAME
367	# undef PGM_GST_TYPE
368	# undef PGM_GST_NAME
369
370	# ifdef VBOX_WITH_64_BITS_GUESTS
371	/* Guest - AMD64 mode */
372	# define PGM_GST_TYPE PGM_TYPE_AMD64
373	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
374	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
375	# include "PGMGstDefs.h"
376	# include "PGMAllBth.h"
377	# undef PGM_BTH_NAME
378	# undef PGM_GST_TYPE
379	# undef PGM_GST_NAME
380	# endif /* VBOX_WITH_64_BITS_GUESTS */
381
382	# undef PGM_SHW_TYPE
383	# undef PGM_SHW_NAME
384
385
386	/*
387	* Shadow - AMD64 nested paging mode.
388	*/
389	# define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
390	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
391	# include "PGMAllShw.h"
392
393	/* Guest - real mode */
394	# define PGM_GST_TYPE PGM_TYPE_REAL
395	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
396	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
397	# include "PGMGstDefs.h"
398	# include "PGMAllBth.h"
399	# undef PGM_BTH_NAME
400	# undef PGM_GST_TYPE
401	# undef PGM_GST_NAME
402
403	/* Guest - protected mode */
404	# define PGM_GST_TYPE PGM_TYPE_PROT
405	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
406	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
407	# include "PGMGstDefs.h"
408	# include "PGMAllBth.h"
409	# undef PGM_BTH_NAME
410	# undef PGM_GST_TYPE
411	# undef PGM_GST_NAME
412
413	/* Guest - 32-bit mode */
414	# define PGM_GST_TYPE PGM_TYPE_32BIT
415	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
416	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
417	# include "PGMGstDefs.h"
418	# include "PGMAllBth.h"
419	# undef PGM_BTH_NAME
420	# undef PGM_GST_TYPE
421	# undef PGM_GST_NAME
422
423	/* Guest - PAE mode */
424	# define PGM_GST_TYPE PGM_TYPE_PAE
425	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
426	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
427	# include "PGMGstDefs.h"
428	# include "PGMAllBth.h"
429	# undef PGM_BTH_NAME
430	# undef PGM_GST_TYPE
431	# undef PGM_GST_NAME
432
433	# ifdef VBOX_WITH_64_BITS_GUESTS
434	/* Guest - AMD64 mode */
435	# define PGM_GST_TYPE PGM_TYPE_AMD64
436	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
437	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
438	# include "PGMGstDefs.h"
439	# include "PGMAllBth.h"
440	# undef PGM_BTH_NAME
441	# undef PGM_GST_TYPE
442	# undef PGM_GST_NAME
443	# endif /* VBOX_WITH_64_BITS_GUESTS */
444
445	# undef PGM_SHW_TYPE
446	# undef PGM_SHW_NAME
447
448
449	/*
450	* Shadow - EPT.
451	*/
452	# define PGM_SHW_TYPE PGM_TYPE_EPT
453	# define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
454	# include "PGMAllShw.h"
455
456	/* Guest - real mode */
457	# define PGM_GST_TYPE PGM_TYPE_REAL
458	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
459	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
460	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
461	# include "PGMGstDefs.h"
462	# include "PGMAllBth.h"
463	# undef BTH_PGMPOOLKIND_PT_FOR_PT
464	# undef PGM_BTH_NAME
465	# undef PGM_GST_TYPE
466	# undef PGM_GST_NAME
467
468	/* Guest - protected mode */
469	# define PGM_GST_TYPE PGM_TYPE_PROT
470	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
471	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
472	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
473	# include "PGMGstDefs.h"
474	# include "PGMAllBth.h"
475	# undef BTH_PGMPOOLKIND_PT_FOR_PT
476	# undef PGM_BTH_NAME
477	# undef PGM_GST_TYPE
478	# undef PGM_GST_NAME
479
480	/* Guest - 32-bit mode */
481	# define PGM_GST_TYPE PGM_TYPE_32BIT
482	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
483	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
484	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
485	# include "PGMGstDefs.h"
486	# include "PGMAllBth.h"
487	# undef BTH_PGMPOOLKIND_PT_FOR_PT
488	# undef PGM_BTH_NAME
489	# undef PGM_GST_TYPE
490	# undef PGM_GST_NAME
491
492	/* Guest - PAE mode */
493	# define PGM_GST_TYPE PGM_TYPE_PAE
494	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
495	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
496	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
497	# include "PGMGstDefs.h"
498	# include "PGMAllBth.h"
499	# undef BTH_PGMPOOLKIND_PT_FOR_PT
500	# undef PGM_BTH_NAME
501	# undef PGM_GST_TYPE
502	# undef PGM_GST_NAME
503
504	# ifdef VBOX_WITH_64_BITS_GUESTS
505	/* Guest - AMD64 mode */
506	# define PGM_GST_TYPE PGM_TYPE_AMD64
507	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
508	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
509	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
510	# include "PGMGstDefs.h"
511	# include "PGMAllBth.h"
512	# undef BTH_PGMPOOLKIND_PT_FOR_PT
513	# undef PGM_BTH_NAME
514	# undef PGM_GST_TYPE
515	# undef PGM_GST_NAME
516	# endif /* VBOX_WITH_64_BITS_GUESTS */
517
518	# undef PGM_SHW_TYPE
519	# undef PGM_SHW_NAME
520
521	#endif /* !IN_RC */
522
523
524	/**
525	* Guest mode data array.
526	*/
527	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
528	{
529	{ UINT32_MAX, NULL, NULL, NULL, NULL, NULL }, /* 0 */
530	{
531	PGM_TYPE_REAL,
532	PGM_GST_NAME_REAL(GetPage),
533	PGM_GST_NAME_REAL(ModifyPage),
534	PGM_GST_NAME_REAL(GetPDE),
535	#ifdef IN_RING3
536	PGM_GST_NAME_REAL(Enter),
537	PGM_GST_NAME_REAL(Exit),
538	PGM_GST_NAME_REAL(Relocate),
539	#else
540	NULL, NULL, NULL,
541	#endif
542	},
543	{
544	PGM_TYPE_PROT,
545	PGM_GST_NAME_PROT(GetPage),
546	PGM_GST_NAME_PROT(ModifyPage),
547	PGM_GST_NAME_PROT(GetPDE),
548	#ifdef IN_RING3
549	PGM_GST_NAME_PROT(Enter),
550	PGM_GST_NAME_PROT(Exit),
551	PGM_GST_NAME_PROT(Relocate),
552	#else
553	NULL, NULL, NULL,
554	#endif
555	},
556	{
557	PGM_TYPE_32BIT,
558	PGM_GST_NAME_32BIT(GetPage),
559	PGM_GST_NAME_32BIT(ModifyPage),
560	PGM_GST_NAME_32BIT(GetPDE),
561	#ifdef IN_RING3
562	PGM_GST_NAME_32BIT(Enter),
563	PGM_GST_NAME_32BIT(Exit),
564	PGM_GST_NAME_32BIT(Relocate),
565	#else
566	NULL, NULL, NULL,
567	#endif
568	},
569	{
570	PGM_TYPE_PAE,
571	PGM_GST_NAME_PAE(GetPage),
572	PGM_GST_NAME_PAE(ModifyPage),
573	PGM_GST_NAME_PAE(GetPDE),
574	#ifdef IN_RING3
575	PGM_GST_NAME_PAE(Enter),
576	PGM_GST_NAME_PAE(Exit),
577	PGM_GST_NAME_PAE(Relocate),
578	#else
579	NULL, NULL, NULL,
580	#endif
581	},
582	#if defined(VBOX_WITH_64_BITS_GUESTS) && !defined(IN_RC)
583	{
584	PGM_TYPE_AMD64,
585	PGM_GST_NAME_AMD64(GetPage),
586	PGM_GST_NAME_AMD64(ModifyPage),
587	PGM_GST_NAME_AMD64(GetPDE),
588	# ifdef IN_RING3
589	PGM_GST_NAME_AMD64(Enter),
590	PGM_GST_NAME_AMD64(Exit),
591	PGM_GST_NAME_AMD64(Relocate),
592	# else
593	NULL, NULL, NULL,
594	# endif
595	},
596	#endif
597	};
598
599
600	/**
601	* The shadow mode data array.
602	*/
603	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
604	{
605	{ UINT8_MAX, NULL, NULL, NULL, NULL, NULL }, /* 0 */
606	{ UINT8_MAX, NULL, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
607	{ UINT8_MAX, NULL, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
608	{
609	PGM_TYPE_32BIT,
610	PGM_SHW_NAME_32BIT(GetPage),
611	PGM_SHW_NAME_32BIT(ModifyPage),
612	#ifdef IN_RING3
613	PGM_SHW_NAME_32BIT(Enter),
614	PGM_SHW_NAME_32BIT(Exit),
615	PGM_SHW_NAME_32BIT(Relocate),
616	#else
617	NULL, NULL, NULL,
618	#endif
619	},
620	{
621	PGM_TYPE_PAE,
622	PGM_SHW_NAME_PAE(GetPage),
623	PGM_SHW_NAME_PAE(ModifyPage),
624	#ifdef IN_RING3
625	PGM_SHW_NAME_PAE(Enter),
626	PGM_SHW_NAME_PAE(Exit),
627	PGM_SHW_NAME_PAE(Relocate),
628	#else
629	NULL, NULL, NULL,
630	#endif
631	},
632	#ifndef IN_RC
633	{
634	PGM_TYPE_AMD64,
635	PGM_SHW_NAME_AMD64(GetPage),
636	PGM_SHW_NAME_AMD64(ModifyPage),
637	# ifdef IN_RING3
638	PGM_SHW_NAME_AMD64(Enter),
639	PGM_SHW_NAME_AMD64(Exit),
640	PGM_SHW_NAME_AMD64(Relocate),
641	# else
642	NULL, NULL, NULL,
643	# endif
644	},
645	{
646	PGM_TYPE_NESTED_32BIT,
647	PGM_SHW_NAME_NESTED_32BIT(GetPage),
648	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
649	# ifdef IN_RING3
650	PGM_SHW_NAME_NESTED_32BIT(Enter),
651	PGM_SHW_NAME_NESTED_32BIT(Exit),
652	PGM_SHW_NAME_NESTED_32BIT(Relocate),
653	# else
654	NULL, NULL, NULL,
655	# endif
656	},
657	{
658	PGM_TYPE_NESTED_PAE,
659	PGM_SHW_NAME_NESTED_PAE(GetPage),
660	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
661	# ifdef IN_RING3
662	PGM_SHW_NAME_NESTED_PAE(Enter),
663	PGM_SHW_NAME_NESTED_PAE(Exit),
664	PGM_SHW_NAME_NESTED_PAE(Relocate),
665	# else
666	NULL, NULL, NULL,
667	# endif
668	},
669	{
670	PGM_TYPE_NESTED_AMD64,
671	PGM_SHW_NAME_NESTED_AMD64(GetPage),
672	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
673	# ifdef IN_RING3
674	PGM_SHW_NAME_NESTED_AMD64(Enter),
675	PGM_SHW_NAME_NESTED_AMD64(Exit),
676	PGM_SHW_NAME_NESTED_AMD64(Relocate),
677	# else
678	NULL, NULL, NULL,
679	# endif
680	},
681	{
682	PGM_TYPE_EPT,
683	PGM_SHW_NAME_EPT(GetPage),
684	PGM_SHW_NAME_EPT(ModifyPage),
685	# ifdef IN_RING3
686	PGM_SHW_NAME_EPT(Enter),
687	PGM_SHW_NAME_EPT(Exit),
688	PGM_SHW_NAME_EPT(Relocate),
689	# else
690	NULL, NULL, NULL,
691	# endif
692	},
693	#endif /* IN_RC */
694	};
695
696
697	/**
698	* The guest+shadow mode data array.
699	*/
700	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
701	{
702	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
703	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
704	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
705	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), /Nm(Enter)/ NULL, Nm(Trap0eHandler) }
706
707	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
708	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
709	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
710	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), /Nm(Enter)/ NULL, Nm(Trap0eHandler), Nm(AssertCR3) }
711
712	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
713	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
714	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
715	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Relocate), }
716
717	#elif defined(IN_RING3) && defined(VBOX_STRICT)
718	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
719	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
720	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Relocate), Nm(AssertCR3) }
721
722	#else
723	# error "Misconfig."
724	#endif
725
726	/* 32-bit shadow paging mode: */
727	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
728	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
729	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
730	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
731	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
732	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
733	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
734	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
735	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
736	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
737
738	/* PAE shadow paging mode: */
739	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
740	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
741	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
742	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
743	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
744	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
745	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
746	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
747	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
748	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
749
750	#ifndef IN_RC
751	/* AMD64 shadow paging mode: */
752	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
753	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
754	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
755	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
756	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
757	# ifdef VBOX_WITH_64_BITS_GUESTS
758	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
759	# else
760	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
761	# endif
762	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
763	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
764	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
765	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
766
767	/* 32-bit nested paging mode: */
768	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
769	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
770	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
771	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
772	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
773	# ifdef VBOX_WITH_64_BITS_GUESTS
774	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
775	# else
776	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
777	# endif
778	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
780	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
781	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
782
783	/* PAE nested paging mode: */
784	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
785	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
786	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
787	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
788	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
789	# ifdef VBOX_WITH_64_BITS_GUESTS
790	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
791	# else
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
793	# endif
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
796	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
797	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
798
799	/* AMD64 nested paging mode: */
800	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
801	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
802	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
803	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
805	# ifdef VBOX_WITH_64_BITS_GUESTS
806	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
807	# else
808	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
809	# endif
810	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
811	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
812	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
813	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
814
815	/* EPT nested paging mode: */
816	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
817	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
818	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
820	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
821	# ifdef VBOX_WITH_64_BITS_GUESTS
822	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
823	# else
824	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
825	# endif
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
827	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
828	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
829	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
830
831	#endif /* IN_RC */
832
833	#undef PGMMODEDATABTH_ENTRY
834	#undef PGMMODEDATABTH_NULL_ENTRY
835	};
836
837
838	#ifndef IN_RING3
839	/**
840	* #PF Handler.
841	*
842	* @returns VBox status code (appropriate for trap handling and GC return).
843	* @param pVCpu The cross context virtual CPU structure.
844	* @param uErr The trap error code.
845	* @param pRegFrame Trap register frame.
846	* @param pvFault The fault address.
847	*/
848	VMMDECL(int) PGMTrap0eHandler(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
849	{
850	PVM pVM = pVCpu->CTX_SUFF(pVM);
851
852	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
853	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, a);
854	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = NULL; } );
855
856
857	#ifdef VBOX_WITH_STATISTICS
858	/*
859	* Error code stats.
860	*/
861	if (uErr & X86_TRAP_PF_US)
862	{
863	if (!(uErr & X86_TRAP_PF_P))
864	{
865	if (uErr & X86_TRAP_PF_RW)
866	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentWrite);
867	else
868	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentRead);
869	}
870	else if (uErr & X86_TRAP_PF_RW)
871	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSWrite);
872	else if (uErr & X86_TRAP_PF_RSVD)
873	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSReserved);
874	else if (uErr & X86_TRAP_PF_ID)
875	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNXE);
876	else
877	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSRead);
878	}
879	else
880	{ /* Supervisor */
881	if (!(uErr & X86_TRAP_PF_P))
882	{
883	if (uErr & X86_TRAP_PF_RW)
884	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentWrite);
885	else
886	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentRead);
887	}
888	else if (uErr & X86_TRAP_PF_RW)
889	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVWrite);
890	else if (uErr & X86_TRAP_PF_ID)
891	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSNXE);
892	else if (uErr & X86_TRAP_PF_RSVD)
893	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVReserved);
894	}
895	#endif /* VBOX_WITH_STATISTICS */
896
897	/*
898	* Call the worker.
899	*/
900	bool fLockTaken = false;
901	int rc = PGM_BTH_PFN(Trap0eHandler, pVCpu)(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
902	if (fLockTaken)
903	{
904	PGM_LOCK_ASSERT_OWNER(pVM);
905	pgmUnlock(pVM);
906	}
907	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
908
909	/*
910	* Return code tweaks.
911	*/
912	if (rc != VINF_SUCCESS)
913	{
914	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
915	rc = VINF_SUCCESS;
916
917	# ifdef IN_RING0
918	/* Note: hack alert for difficult to reproduce problem. */
919	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
920	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
921	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
922	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
923	{
924	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
925	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
926	rc = VINF_SUCCESS;
927	}
928	# endif
929	}
930
931	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPF); });
932	STAM_STATS({ if (!pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution))
933	pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Misc; });
934	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution), a);
935	return rc;
936	}
937	#endif /* !IN_RING3 */
938
939
940	/**
941	* Prefetch a page
942	*
943	* Typically used to sync commonly used pages before entering raw mode
944	* after a CR3 reload.
945	*
946	* @returns VBox status code suitable for scheduling.
947	* @retval VINF_SUCCESS on success.
948	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
949	* @param pVCpu The cross context virtual CPU structure.
950	* @param GCPtrPage Page to invalidate.
951	*/
952	VMMDECL(int) PGMPrefetchPage(PVMCPU pVCpu, RTGCPTR GCPtrPage)
953	{
954	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
955	int rc = PGM_BTH_PFN(PrefetchPage, pVCpu)(pVCpu, GCPtrPage);
956	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
957	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
958	return rc;
959	}
960
961
962	/**
963	* Gets the mapping corresponding to the specified address (if any).
964	*
965	* @returns Pointer to the mapping.
966	* @returns NULL if not
967	*
968	* @param pVM The cross context VM structure.
969	* @param GCPtr The guest context pointer.
970	*/
971	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr)
972	{
973	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
974	while (pMapping)
975	{
976	if ((uintptr_t)GCPtr < (uintptr_t)pMapping->GCPtr)
977	break;
978	if ((uintptr_t)GCPtr - (uintptr_t)pMapping->GCPtr < pMapping->cb)
979	return pMapping;
980	pMapping = pMapping->CTX_SUFF(pNext);
981	}
982	return NULL;
983	}
984
985
986	/**
987	* Verifies a range of pages for read or write access
988	*
989	* Only checks the guest's page tables
990	*
991	* @returns VBox status code.
992	* @param pVCpu The cross context virtual CPU structure.
993	* @param Addr Guest virtual address to check
994	* @param cbSize Access size
995	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
996	* @remarks Current not in use.
997	*/
998	VMMDECL(int) PGMIsValidAccess(PVMCPU pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
999	{
1000	/*
1001	* Validate input.
1002	*/
1003	if (fAccess & ~(X86_PTE_US \| X86_PTE_RW))
1004	{
1005	AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
1006	return VERR_INVALID_PARAMETER;
1007	}
1008
1009	uint64_t fPage;
1010	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPage, NULL);
1011	if (RT_FAILURE(rc))
1012	{
1013	Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
1014	return VINF_EM_RAW_GUEST_TRAP;
1015	}
1016
1017	/*
1018	* Check if the access would cause a page fault
1019	*
1020	* Note that hypervisor page directories are not present in the guest's tables, so this check
1021	* is sufficient.
1022	*/
1023	bool fWrite = !!(fAccess & X86_PTE_RW);
1024	bool fUser = !!(fAccess & X86_PTE_US);
1025	if ( !(fPage & X86_PTE_P)
1026	\|\| (fWrite && !(fPage & X86_PTE_RW))
1027	\|\| (fUser && !(fPage & X86_PTE_US)) )
1028	{
1029	Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
1030	return VINF_EM_RAW_GUEST_TRAP;
1031	}
1032	if ( RT_SUCCESS(rc)
1033	&& PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
1034	return PGMIsValidAccess(pVCpu, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
1035	return rc;
1036	}
1037
1038
1039	/**
1040	* Verifies a range of pages for read or write access
1041	*
1042	* Supports handling of pages marked for dirty bit tracking and CSAM
1043	*
1044	* @returns VBox status code.
1045	* @param pVCpu The cross context virtual CPU structure.
1046	* @param Addr Guest virtual address to check
1047	* @param cbSize Access size
1048	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1049	*/
1050	VMMDECL(int) PGMVerifyAccess(PVMCPU pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1051	{
1052	PVM pVM = pVCpu->CTX_SUFF(pVM);
1053
1054	AssertMsg(!(fAccess & ~(X86_PTE_US \| X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
1055
1056	/*
1057	* Get going.
1058	*/
1059	uint64_t fPageGst;
1060	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPageGst, NULL);
1061	if (RT_FAILURE(rc))
1062	{
1063	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
1064	return VINF_EM_RAW_GUEST_TRAP;
1065	}
1066
1067	/*
1068	* Check if the access would cause a page fault
1069	*
1070	* Note that hypervisor page directories are not present in the guest's tables, so this check
1071	* is sufficient.
1072	*/
1073	const bool fWrite = !!(fAccess & X86_PTE_RW);
1074	const bool fUser = !!(fAccess & X86_PTE_US);
1075	if ( !(fPageGst & X86_PTE_P)
1076	\|\| (fWrite && !(fPageGst & X86_PTE_RW))
1077	\|\| (fUser && !(fPageGst & X86_PTE_US)) )
1078	{
1079	Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
1080	return VINF_EM_RAW_GUEST_TRAP;
1081	}
1082
1083	if (!pVM->pgm.s.fNestedPaging)
1084	{
1085	/*
1086	* Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
1087	*/
1088	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, NULL, NULL);
1089	if ( rc == VERR_PAGE_NOT_PRESENT
1090	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT)
1091	{
1092	/*
1093	* Page is not present in our page tables.
1094	* Try to sync it!
1095	*/
1096	Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
1097	uint32_t uErr = fAccess & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
1098	rc = PGM_BTH_PFN(VerifyAccessSyncPage, pVCpu)(pVCpu, Addr, fPageGst, uErr);
1099	if (rc != VINF_SUCCESS)
1100	return rc;
1101	}
1102	else
1103	AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
1104	}
1105
1106	#if 0 /* def VBOX_STRICT; triggers too often now */
1107	/*
1108	* This check is a bit paranoid, but useful.
1109	*/
1110	/* Note! This will assert when writing to monitored pages (a bit annoying actually). */
1111	uint64_t fPageShw;
1112	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, &fPageShw, NULL);
1113	if ( (rc == VERR_PAGE_NOT_PRESENT \|\| RT_FAILURE(rc))
1114	\|\| (fWrite && !(fPageShw & X86_PTE_RW))
1115	\|\| (fUser && !(fPageShw & X86_PTE_US)) )
1116	{
1117	AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
1118	Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
1119	return VINF_EM_RAW_GUEST_TRAP;
1120	}
1121	#endif
1122
1123	if ( RT_SUCCESS(rc)
1124	&& ( PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
1125	\|\| Addr + cbSize < Addr))
1126	{
1127	/* Don't recursively call PGMVerifyAccess as we might run out of stack. */
1128	for (;;)
1129	{
1130	Addr += PAGE_SIZE;
1131	if (cbSize > PAGE_SIZE)
1132	cbSize -= PAGE_SIZE;
1133	else
1134	cbSize = 1;
1135	rc = PGMVerifyAccess(pVCpu, Addr, 1, fAccess);
1136	if (rc != VINF_SUCCESS)
1137	break;
1138	if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
1139	break;
1140	}
1141	}
1142	return rc;
1143	}
1144
1145
1146	/**
1147	* Emulation of the invlpg instruction (HC only actually).
1148	*
1149	* @returns Strict VBox status code, special care required.
1150	* @retval VINF_PGM_SYNC_CR3 - handled.
1151	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1152	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1153	*
1154	* @param pVCpu The cross context virtual CPU structure.
1155	* @param GCPtrPage Page to invalidate.
1156	*
1157	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1158	* safe, but there could be edge cases!
1159	*
1160	* @todo Flush page or page directory only if necessary!
1161	* @todo VBOXSTRICTRC
1162	*/
1163	VMMDECL(int) PGMInvalidatePage(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1164	{
1165	PVM pVM = pVCpu->CTX_SUFF(pVM);
1166	int rc;
1167	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1168
1169	#if !defined(IN_RING3) && defined(VBOX_WITH_REM)
1170	/*
1171	* Notify the recompiler so it can record this instruction.
1172	*/
1173	REMNotifyInvalidatePage(pVM, GCPtrPage);
1174	#endif /* !IN_RING3 */
1175	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1176
1177
1178	#ifdef IN_RC
1179	/*
1180	* Check for conflicts and pending CR3 monitoring updates.
1181	*/
1182	if (pgmMapAreMappingsFloating(pVM))
1183	{
1184	if ( pgmGetMapping(pVM, GCPtrPage)
1185	&& PGMGstGetPage(pVCpu, GCPtrPage, NULL, NULL) != VERR_PAGE_TABLE_NOT_PRESENT)
1186	{
1187	LogFlow(("PGMGCInvalidatePage: Conflict!\n"));
1188	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
1189	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRCInvlPgConflict);
1190	return VINF_PGM_SYNC_CR3;
1191	}
1192
1193	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
1194	{
1195	LogFlow(("PGMGCInvalidatePage: PGM_SYNC_MONITOR_CR3 -> reinterpret instruction in R3\n"));
1196	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatRCInvlPgSyncMonCR3);
1197	return VINF_EM_RAW_EMULATE_INSTR;
1198	}
1199	}
1200	#endif /* IN_RC */
1201
1202	/*
1203	* Call paging mode specific worker.
1204	*/
1205	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
1206	pgmLock(pVM);
1207	rc = PGM_BTH_PFN(InvalidatePage, pVCpu)(pVCpu, GCPtrPage);
1208	pgmUnlock(pVM);
1209	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
1210
1211	#ifdef IN_RING3
1212	/*
1213	* Check if we have a pending update of the CR3 monitoring.
1214	*/
1215	if ( RT_SUCCESS(rc)
1216	&& (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3))
1217	{
1218	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1219	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
1220	}
1221
1222	# ifdef VBOX_WITH_RAW_MODE
1223	/*
1224	* Inform CSAM about the flush
1225	*
1226	* Note: This is to check if monitored pages have been changed; when we implement
1227	* callbacks for virtual handlers, this is no longer required.
1228	*/
1229	CSAMR3FlushPage(pVM, GCPtrPage);
1230	# endif
1231	#endif /* IN_RING3 */
1232
1233	/* Ignore all irrelevant error codes. */
1234	if ( rc == VERR_PAGE_NOT_PRESENT
1235	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1236	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1237	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1238	rc = VINF_SUCCESS;
1239
1240	return rc;
1241	}
1242
1243
1244	/**
1245	* Executes an instruction using the interpreter.
1246	*
1247	* @returns VBox status code (appropriate for trap handling and GC return).
1248	* @param pVM The cross context VM structure.
1249	* @param pVCpu The cross context virtual CPU structure.
1250	* @param pRegFrame Register frame.
1251	* @param pvFault Fault address.
1252	*/
1253	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
1254	{
1255	NOREF(pVM);
1256	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
1257	if (rc == VERR_EM_INTERPRETER)
1258	rc = VINF_EM_RAW_EMULATE_INSTR;
1259	if (rc != VINF_SUCCESS)
1260	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1261	return rc;
1262	}
1263
1264
1265	/**
1266	* Gets effective page information (from the VMM page directory).
1267	*
1268	* @returns VBox status code.
1269	* @param pVCpu The cross context virtual CPU structure.
1270	* @param GCPtr Guest Context virtual address of the page.
1271	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1272	* @param pHCPhys Where to store the HC physical address of the page.
1273	* This is page aligned.
1274	* @remark You should use PGMMapGetPage() for pages in a mapping.
1275	*/
1276	VMMDECL(int) PGMShwGetPage(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1277	{
1278	PVM pVM = pVCpu->CTX_SUFF(pVM);
1279	pgmLock(pVM);
1280
1281	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1282	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1283	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1284	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1285
1286	pgmUnlock(pVM);
1287	return rc;
1288	}
1289
1290
1291	/**
1292	* Modify page flags for a range of pages in the shadow context.
1293	*
1294	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1295	*
1296	* @returns VBox status code.
1297	* @param pVCpu The cross context virtual CPU structure.
1298	* @param GCPtr Virtual address of the first page in the range.
1299	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1300	* @param fMask The AND mask - page flags X86_PTE_*.
1301	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1302	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1303	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1304	*/
1305	DECLINLINE(int) pdmShwModifyPage(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1306	{
1307	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1308	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1309
1310	GCPtr &= PAGE_BASE_GC_MASK; /** @todo this ain't necessary, right... */
1311
1312	PVM pVM = pVCpu->CTX_SUFF(pVM);
1313	pgmLock(pVM);
1314
1315	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1316	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1317	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1318	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, PAGE_SIZE, fFlags, fMask, fOpFlags);
1319
1320	pgmUnlock(pVM);
1321	return rc;
1322	}
1323
1324
1325	/**
1326	* Changing the page flags for a single page in the shadow page tables so as to
1327	* make it read-only.
1328	*
1329	* @returns VBox status code.
1330	* @param pVCpu The cross context virtual CPU structure.
1331	* @param GCPtr Virtual address of the first page in the range.
1332	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1333	*/
1334	VMMDECL(int) PGMShwMakePageReadonly(PVMCPU pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1335	{
1336	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1337	}
1338
1339
1340	/**
1341	* Changing the page flags for a single page in the shadow page tables so as to
1342	* make it writable.
1343	*
1344	* The call must know with 101% certainty that the guest page tables maps this
1345	* as writable too. This function will deal shared, zero and write monitored
1346	* pages.
1347	*
1348	* @returns VBox status code.
1349	* @param pVCpu The cross context virtual CPU structure.
1350	* @param GCPtr Virtual address of the first page in the range.
1351	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1352	*/
1353	VMMDECL(int) PGMShwMakePageWritable(PVMCPU pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1354	{
1355	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1356	}
1357
1358
1359	/**
1360	* Changing the page flags for a single page in the shadow page tables so as to
1361	* make it not present.
1362	*
1363	* @returns VBox status code.
1364	* @param pVCpu The cross context virtual CPU structure.
1365	* @param GCPtr Virtual address of the first page in the range.
1366	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1367	*/
1368	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPU pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1369	{
1370	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1371	}
1372
1373
1374	/**
1375	* Changing the page flags for a single page in the shadow page tables so as to
1376	* make it supervisor and writable.
1377	*
1378	* This if for dealing with CR0.WP=0 and readonly user pages.
1379	*
1380	* @returns VBox status code.
1381	* @param pVCpu The cross context virtual CPU structure.
1382	* @param GCPtr Virtual address of the first page in the range.
1383	* @param fBigPage Whether or not this is a big page. If it is, we have to
1384	* change the shadow PDE as well. If it isn't, the caller
1385	* has checked that the shadow PDE doesn't need changing.
1386	* We ASSUME 4KB pages backing the big page here!
1387	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1388	*/
1389	int pgmShwMakePageSupervisorAndWritable(PVMCPU pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1390	{
1391	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1392	if (rc == VINF_SUCCESS && fBigPage)
1393	{
1394	/* this is a bit ugly... */
1395	switch (pVCpu->pgm.s.enmShadowMode)
1396	{
1397	case PGMMODE_32_BIT:
1398	{
1399	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1400	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1401	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1402	pPde->n.u1Write = 1;
1403	Log(("-> PDE=%#llx (32)\n", pPde->u));
1404	break;
1405	}
1406	case PGMMODE_PAE:
1407	case PGMMODE_PAE_NX:
1408	{
1409	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1410	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1411	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1412	pPde->n.u1Write = 1;
1413	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1414	break;
1415	}
1416	default:
1417	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1418	}
1419	}
1420	return rc;
1421	}
1422
1423
1424	/**
1425	* Gets the shadow page directory for the specified address, PAE.
1426	*
1427	* @returns Pointer to the shadow PD.
1428	* @param pVCpu The cross context virtual CPU structure.
1429	* @param GCPtr The address.
1430	* @param uGstPdpe Guest PDPT entry. Valid.
1431	* @param ppPD Receives address of page directory
1432	*/
1433	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1434	{
1435	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1436	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1437	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1438	PVM pVM = pVCpu->CTX_SUFF(pVM);
1439	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1440	PPGMPOOLPAGE pShwPage;
1441	int rc;
1442
1443	PGM_LOCK_ASSERT_OWNER(pVM);
1444
1445	/* Allocate page directory if not present. */
1446	if ( !pPdpe->n.u1Present
1447	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1448	{
1449	RTGCPTR64 GCPdPt;
1450	PGMPOOLKIND enmKind;
1451
1452	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1453	{
1454	/* AMD-V nested paging or real/protected mode without paging. */
1455	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1456	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1457	}
1458	else
1459	{
1460	if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1461	{
1462	if (!(uGstPdpe & X86_PDPE_P))
1463	{
1464	/* PD not present; guest must reload CR3 to change it.
1465	* No need to monitor anything in this case.
1466	*/
1467	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1468
1469	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1470	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1471	uGstPdpe \|= X86_PDPE_P;
1472	}
1473	else
1474	{
1475	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1476	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1477	}
1478	}
1479	else
1480	{
1481	GCPdPt = CPUMGetGuestCR3(pVCpu);
1482	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1483	}
1484	}
1485
1486	/* Create a reference back to the PDPT by using the index in its shadow page. */
1487	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1488	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1489	&pShwPage);
1490	AssertRCReturn(rc, rc);
1491
1492	/* The PD was cached or created; hook it up now. */
1493	pPdpe->u \|= pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A));
1494
1495	# if defined(IN_RC)
1496	/*
1497	* In 32 bits PAE mode we must invalidate the TLB when changing a
1498	* PDPT entry; the CPU fetches them only during cr3 load, so any
1499	* non-present PDPT will continue to cause page faults.
1500	*/
1501	ASMReloadCR3();
1502	# endif
1503	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1504	}
1505	else
1506	{
1507	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1508	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1509	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1510
1511	pgmPoolCacheUsed(pPool, pShwPage);
1512	}
1513	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	/**
1519	* Gets the pointer to the shadow page directory entry for an address, PAE.
1520	*
1521	* @returns Pointer to the PDE.
1522	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1523	* @param GCPtr The address.
1524	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1525	*/
1526	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPU pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1527	{
1528	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1529	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1530	PVM pVM = pVCpu->CTX_SUFF(pVM);
1531
1532	PGM_LOCK_ASSERT_OWNER(pVM);
1533
1534	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1535	if (!pPdpt->a[iPdPt].n.u1Present)
1536	{
1537	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, pPdpt->a[iPdPt].u));
1538	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1539	}
1540	AssertMsg(pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1541
1542	/* Fetch the pgm pool shadow descriptor. */
1543	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1544	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1545
1546	*ppShwPde = pShwPde;
1547	return VINF_SUCCESS;
1548	}
1549
1550	#ifndef IN_RC
1551
1552	/**
1553	* Syncs the SHADOW page directory pointer for the specified address.
1554	*
1555	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1556	*
1557	* The caller is responsible for making sure the guest has a valid PD before
1558	* calling this function.
1559	*
1560	* @returns VBox status code.
1561	* @param pVCpu The cross context virtual CPU structure.
1562	* @param GCPtr The address.
1563	* @param uGstPml4e Guest PML4 entry (valid).
1564	* @param uGstPdpe Guest PDPT entry (valid).
1565	* @param ppPD Receives address of page directory
1566	*/
1567	static int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1568	{
1569	PVM pVM = pVCpu->CTX_SUFF(pVM);
1570	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1571	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1572	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1573	bool fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1574	PPGMPOOLPAGE pShwPage;
1575	int rc;
1576
1577	PGM_LOCK_ASSERT_OWNER(pVM);
1578
1579	/* Allocate page directory pointer table if not present. */
1580	if ( !pPml4e->n.u1Present
1581	&& !(pPml4e->u & X86_PML4E_PG_MASK))
1582	{
1583	RTGCPTR64 GCPml4;
1584	PGMPOOLKIND enmKind;
1585
1586	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1587
1588	if (fNestedPagingOrNoGstPaging)
1589	{
1590	/* AMD-V nested paging or real/protected mode without paging */
1591	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
1592	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1593	}
1594	else
1595	{
1596	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1597	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1598	}
1599
1600	/* Create a reference back to the PDPT by using the index in its shadow page. */
1601	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1602	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1603	&pShwPage);
1604	AssertRCReturn(rc, rc);
1605	}
1606	else
1607	{
1608	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1609	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1610
1611	pgmPoolCacheUsed(pPool, pShwPage);
1612	}
1613	/* The PDPT was cached or created; hook it up now. */
1614	pPml4e->u \|= pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask);
1615
1616	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1617	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1618	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1619
1620	/* Allocate page directory if not present. */
1621	if ( !pPdpe->n.u1Present
1622	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1623	{
1624	RTGCPTR64 GCPdPt;
1625	PGMPOOLKIND enmKind;
1626
1627	if (fNestedPagingOrNoGstPaging)
1628	{
1629	/* AMD-V nested paging or real/protected mode without paging */
1630	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1631	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1632	}
1633	else
1634	{
1635	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1636	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1637	}
1638
1639	/* Create a reference back to the PDPT by using the index in its shadow page. */
1640	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1641	pShwPage->idx, iPdPt, false /fLockPage/,
1642	&pShwPage);
1643	AssertRCReturn(rc, rc);
1644	}
1645	else
1646	{
1647	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1648	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1649
1650	pgmPoolCacheUsed(pPool, pShwPage);
1651	}
1652	/* The PD was cached or created; hook it up now. */
1653	pPdpe->u \|= pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask);
1654
1655	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1656	return VINF_SUCCESS;
1657	}
1658
1659
1660	/**
1661	* Gets the SHADOW page directory pointer for the specified address (long mode).
1662	*
1663	* @returns VBox status code.
1664	* @param pVCpu The cross context virtual CPU structure.
1665	* @param GCPtr The address.
1666	* @param ppPdpt Receives address of pdpt
1667	* @param ppPD Receives address of page directory
1668	*/
1669	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1670	{
1671	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1672	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1673
1674	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1675
1676	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1677	if (ppPml4e)
1678	*ppPml4e = (PX86PML4E)pPml4e;
1679
1680	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, pPml4e->u));
1681
1682	if (!pPml4e->n.u1Present)
1683	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1684
1685	PVM pVM = pVCpu->CTX_SUFF(pVM);
1686	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1687	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1688	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1689
1690	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1691	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1692	if (!pPdpt->a[iPdPt].n.u1Present)
1693	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1694
1695	pShwPage = pgmPoolGetPage(pPool, pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1696	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1697
1698	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1699	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1700	return VINF_SUCCESS;
1701	}
1702
1703
1704	/**
1705	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1706	* backing pages in case the PDPT or PML4 entry is missing.
1707	*
1708	* @returns VBox status code.
1709	* @param pVCpu The cross context virtual CPU structure.
1710	* @param GCPtr The address.
1711	* @param ppPdpt Receives address of pdpt
1712	* @param ppPD Receives address of page directory
1713	*/
1714	static int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1715	{
1716	PVM pVM = pVCpu->CTX_SUFF(pVM);
1717	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1718	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1719	PEPTPML4 pPml4;
1720	PEPTPML4E pPml4e;
1721	PPGMPOOLPAGE pShwPage;
1722	int rc;
1723
1724	Assert(pVM->pgm.s.fNestedPaging);
1725	PGM_LOCK_ASSERT_OWNER(pVM);
1726
1727	pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1728	Assert(pPml4);
1729
1730	/* Allocate page directory pointer table if not present. */
1731	pPml4e = &pPml4->a[iPml4];
1732	if ( !pPml4e->n.u1Present
1733	&& !(pPml4e->u & EPT_PML4E_PG_MASK))
1734	{
1735	Assert(!(pPml4e->u & EPT_PML4E_PG_MASK));
1736	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1737
1738	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1739	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1740	&pShwPage);
1741	AssertRCReturn(rc, rc);
1742	}
1743	else
1744	{
1745	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1746	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1747
1748	pgmPoolCacheUsed(pPool, pShwPage);
1749	}
1750	/* The PDPT was cached or created; hook it up now and fill with the default value. */
1751	pPml4e->u = pShwPage->Core.Key;
1752	pPml4e->n.u1Present = 1;
1753	pPml4e->n.u1Write = 1;
1754	pPml4e->n.u1Execute = 1;
1755
1756	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1757	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1758	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1759
1760	if (ppPdpt)
1761	*ppPdpt = pPdpt;
1762
1763	/* Allocate page directory if not present. */
1764	if ( !pPdpe->n.u1Present
1765	&& !(pPdpe->u & EPT_PDPTE_PG_MASK))
1766	{
1767	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1768	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1769	pShwPage->idx, iPdPt, false /fLockPage/,
1770	&pShwPage);
1771	AssertRCReturn(rc, rc);
1772	}
1773	else
1774	{
1775	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1776	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1777
1778	pgmPoolCacheUsed(pPool, pShwPage);
1779	}
1780	/* The PD was cached or created; hook it up now and fill with the default value. */
1781	pPdpe->u = pShwPage->Core.Key;
1782	pPdpe->n.u1Present = 1;
1783	pPdpe->n.u1Write = 1;
1784	pPdpe->n.u1Execute = 1;
1785
1786	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1787	return VINF_SUCCESS;
1788	}
1789
1790	#endif /* IN_RC */
1791
1792	#ifdef IN_RING0
1793	/**
1794	* Synchronizes a range of nested page table entries.
1795	*
1796	* The caller must own the PGM lock.
1797	*
1798	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1799	* @param GCPhys Where to start.
1800	* @param cPages How many pages which entries should be synced.
1801	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1802	* host paging mode for AMD-V).
1803	*/
1804	int pgmShwSyncNestedPageLocked(PVMCPU pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1805	{
1806	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1807
1808	int rc;
1809	switch (enmShwPagingMode)
1810	{
1811	case PGMMODE_32_BIT:
1812	{
1813	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1814	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1815	break;
1816	}
1817
1818	case PGMMODE_PAE:
1819	case PGMMODE_PAE_NX:
1820	{
1821	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1822	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1823	break;
1824	}
1825
1826	case PGMMODE_AMD64:
1827	case PGMMODE_AMD64_NX:
1828	{
1829	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1830	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1831	break;
1832	}
1833
1834	case PGMMODE_EPT:
1835	{
1836	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1837	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1838	break;
1839	}
1840
1841	default:
1842	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1843	}
1844	return rc;
1845	}
1846	#endif /* IN_RING0 */
1847
1848
1849	/**
1850	* Gets effective Guest OS page information.
1851	*
1852	* When GCPtr is in a big page, the function will return as if it was a normal
1853	* 4KB page. If the need for distinguishing between big and normal page becomes
1854	* necessary at a later point, a PGMGstGetPage() will be created for that
1855	* purpose.
1856	*
1857	* @returns VBox status code.
1858	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1859	* @param GCPtr Guest Context virtual address of the page.
1860	* @param pfFlags Where to store the flags. These are X86_PTE_*, even for big pages.
1861	* @param pGCPhys Where to store the GC physical address of the page.
1862	* This is page aligned. The fact that the
1863	*/
1864	VMMDECL(int) PGMGstGetPage(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
1865	{
1866	VMCPU_ASSERT_EMT(pVCpu);
1867	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1868	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1869	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1870	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pfFlags, pGCPhys);
1871	}
1872
1873
1874	/**
1875	* Performs a guest page table walk.
1876	*
1877	* The guest should be in paged protect mode or long mode when making a call to
1878	* this function.
1879	*
1880	* @returns VBox status code.
1881	* @retval VINF_SUCCESS on success.
1882	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1883	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1884	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1885	*
1886	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1887	* @param GCPtr The guest virtual address to walk by.
1888	* @param pWalk Where to return the walk result. This is valid for some
1889	* error codes as well.
1890	*/
1891	int pgmGstPtWalk(PVMCPU pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1892	{
1893	VMCPU_ASSERT_EMT(pVCpu);
1894	switch (pVCpu->pgm.s.enmGuestMode)
1895	{
1896	case PGMMODE_32_BIT:
1897	pWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1898	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, &pWalk->u.Legacy);
1899
1900	case PGMMODE_PAE:
1901	case PGMMODE_PAE_NX:
1902	pWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1903	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, &pWalk->u.Pae);
1904
1905	#if !defined(IN_RC)
1906	case PGMMODE_AMD64:
1907	case PGMMODE_AMD64_NX:
1908	pWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
1909	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, &pWalk->u.Amd64);
1910	#endif
1911
1912	case PGMMODE_REAL:
1913	case PGMMODE_PROTECTED:
1914	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1915	return VERR_PGM_NOT_USED_IN_MODE;
1916
1917	#if defined(IN_RC)
1918	case PGMMODE_AMD64:
1919	case PGMMODE_AMD64_NX:
1920	#endif
1921	case PGMMODE_NESTED_32BIT:
1922	case PGMMODE_NESTED_PAE:
1923	case PGMMODE_NESTED_AMD64:
1924	case PGMMODE_EPT:
1925	default:
1926	AssertFailed();
1927	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1928	return VERR_PGM_NOT_USED_IN_MODE;
1929	}
1930	}
1931
1932
1933	/**
1934	* Tries to continue the previous walk.
1935	*
1936	* @note Requires the caller to hold the PGM lock from the first
1937	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
1938	* we cannot use the pointers.
1939	*
1940	* @returns VBox status code.
1941	* @retval VINF_SUCCESS on success.
1942	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1943	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1944	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1945	*
1946	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1947	* @param GCPtr The guest virtual address to walk by.
1948	* @param pWalk Pointer to the previous walk result and where to return
1949	* the result of this walk. This is valid for some error
1950	* codes as well.
1951	*/
1952	int pgmGstPtWalkNext(PVMCPU pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1953	{
1954	/*
1955	* We can only handle successfully walks.
1956	* We also limit ourselves to the next page.
1957	*/
1958	if ( pWalk->u.Core.fSucceeded
1959	&& GCPtr - pWalk->u.Core.GCPtr == PAGE_SIZE)
1960	{
1961	Assert(pWalk->u.Core.uLevel == 0);
1962	if (pWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
1963	{
1964	/*
1965	* AMD64
1966	*/
1967	if (!pWalk->u.Core.fGigantPage && !pWalk->u.Core.fBigPage)
1968	{
1969	/*
1970	* We fall back to full walk if the PDE table changes, if any
1971	* reserved bits are set, or if the effective page access changes.
1972	*/
1973	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
1974	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
1975	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
1976	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
1977
1978	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PD_PAE_SHIFT))
1979	{
1980	if (pWalk->u.Amd64.pPte)
1981	{
1982	X86PTEPAE Pte;
1983	Pte.u = pWalk->u.Amd64.pPte[1].u;
1984	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
1985	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
1986	{
1987
1988	pWalk->u.Core.GCPtr = GCPtr;
1989	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
1990	pWalk->u.Amd64.Pte.u = Pte.u;
1991	pWalk->u.Amd64.pPte++;
1992	return VINF_SUCCESS;
1993	}
1994	}
1995	}
1996	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PDPT_SHIFT))
1997	{
1998	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
1999	if (pWalk->u.Amd64.pPde)
2000	{
2001	X86PDEPAE Pde;
2002	Pde.u = pWalk->u.Amd64.pPde[1].u;
2003	if ( (Pde.u & fPdeSame) == (pWalk->u.Amd64.Pde.u & fPdeSame)
2004	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2005	{
2006	/* Get the new PTE and check out the first entry. */
2007	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2008	&pWalk->u.Amd64.pPt);
2009	if (RT_SUCCESS(rc))
2010	{
2011	pWalk->u.Amd64.pPte = &pWalk->u.Amd64.pPt->a[0];
2012	X86PTEPAE Pte;
2013	Pte.u = pWalk->u.Amd64.pPte->u;
2014	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
2015	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2016	{
2017	pWalk->u.Core.GCPtr = GCPtr;
2018	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2019	pWalk->u.Amd64.Pte.u = Pte.u;
2020	pWalk->u.Amd64.Pde.u = Pde.u;
2021	pWalk->u.Amd64.pPde++;
2022	return VINF_SUCCESS;
2023	}
2024	}
2025	}
2026	}
2027	}
2028	}
2029	else if (!pWalk->u.Core.fGigantPage)
2030	{
2031	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_2M_BASE_MASK))
2032	{
2033	pWalk->u.Core.GCPtr = GCPtr;
2034	pWalk->u.Core.GCPhys += PAGE_SIZE;
2035	return VINF_SUCCESS;
2036	}
2037	}
2038	else
2039	{
2040	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_1G_BASE_MASK))
2041	{
2042	pWalk->u.Core.GCPtr = GCPtr;
2043	pWalk->u.Core.GCPhys += PAGE_SIZE;
2044	return VINF_SUCCESS;
2045	}
2046	}
2047	}
2048	}
2049	/* Case we don't handle. Do full walk. */
2050	return pgmGstPtWalk(pVCpu, GCPtr, pWalk);
2051	}
2052
2053
2054	/**
2055	* Checks if the page is present.
2056	*
2057	* @returns true if the page is present.
2058	* @returns false if the page is not present.
2059	* @param pVCpu The cross context virtual CPU structure.
2060	* @param GCPtr Address within the page.
2061	*/
2062	VMMDECL(bool) PGMGstIsPagePresent(PVMCPU pVCpu, RTGCPTR GCPtr)
2063	{
2064	VMCPU_ASSERT_EMT(pVCpu);
2065	int rc = PGMGstGetPage(pVCpu, GCPtr, NULL, NULL);
2066	return RT_SUCCESS(rc);
2067	}
2068
2069
2070	/**
2071	* Sets (replaces) the page flags for a range of pages in the guest's tables.
2072	*
2073	* @returns VBox status code.
2074	* @param pVCpu The cross context virtual CPU structure.
2075	* @param GCPtr The address of the first page.
2076	* @param cb The size of the range in bytes.
2077	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
2078	*/
2079	VMMDECL(int) PGMGstSetPage(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
2080	{
2081	VMCPU_ASSERT_EMT(pVCpu);
2082	return PGMGstModifyPage(pVCpu, GCPtr, cb, fFlags, 0);
2083	}
2084
2085
2086	/**
2087	* Modify page flags for a range of pages in the guest's tables
2088	*
2089	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2090	*
2091	* @returns VBox status code.
2092	* @param pVCpu The cross context virtual CPU structure.
2093	* @param GCPtr Virtual address of the first page in the range.
2094	* @param cb Size (in bytes) of the range to apply the modification to.
2095	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2096	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2097	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2098	*/
2099	VMMDECL(int) PGMGstModifyPage(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2100	{
2101	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
2102	VMCPU_ASSERT_EMT(pVCpu);
2103
2104	/*
2105	* Validate input.
2106	*/
2107	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2108	Assert(cb);
2109
2110	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2111
2112	/*
2113	* Adjust input.
2114	*/
2115	cb += GCPtr & PAGE_OFFSET_MASK;
2116	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
2117	GCPtr = (GCPtr & PAGE_BASE_GC_MASK);
2118
2119	/*
2120	* Call worker.
2121	*/
2122	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2123	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2124	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2125	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2126
2127	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
2128	return rc;
2129	}
2130
2131
2132	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2133
2134	/**
2135	* Performs the lazy mapping of the 32-bit guest PD.
2136	*
2137	* @returns VBox status code.
2138	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2139	* @param ppPd Where to return the pointer to the mapping. This is
2140	* always set.
2141	*/
2142	int pgmGstLazyMap32BitPD(PVMCPU pVCpu, PX86PD *ppPd)
2143	{
2144	PVM pVM = pVCpu->CTX_SUFF(pVM);
2145	pgmLock(pVM);
2146
2147	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2148
2149	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
2150	PPGMPAGE pPage;
2151	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2152	if (RT_SUCCESS(rc))
2153	{
2154	RTHCPTR HCPtrGuestCR3;
2155	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2156	if (RT_SUCCESS(rc))
2157	{
2158	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
2159	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2160	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
2161	# endif
2162	*ppPd = (PX86PD)HCPtrGuestCR3;
2163
2164	pgmUnlock(pVM);
2165	return VINF_SUCCESS;
2166	}
2167
2168	AssertRC(rc);
2169	}
2170	pgmUnlock(pVM);
2171
2172	*ppPd = NULL;
2173	return rc;
2174	}
2175
2176
2177	/**
2178	* Performs the lazy mapping of the PAE guest PDPT.
2179	*
2180	* @returns VBox status code.
2181	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2182	* @param ppPdpt Where to return the pointer to the mapping. This is
2183	* always set.
2184	*/
2185	int pgmGstLazyMapPaePDPT(PVMCPU pVCpu, PX86PDPT *ppPdpt)
2186	{
2187	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2188	PVM pVM = pVCpu->CTX_SUFF(pVM);
2189	pgmLock(pVM);
2190
2191	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
2192	PPGMPAGE pPage;
2193	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2194	if (RT_SUCCESS(rc))
2195	{
2196	RTHCPTR HCPtrGuestCR3;
2197	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2198	if (RT_SUCCESS(rc))
2199	{
2200	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
2201	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2202	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
2203	# endif
2204	*ppPdpt = (PX86PDPT)HCPtrGuestCR3;
2205
2206	pgmUnlock(pVM);
2207	return VINF_SUCCESS;
2208	}
2209
2210	AssertRC(rc);
2211	}
2212
2213	pgmUnlock(pVM);
2214	*ppPdpt = NULL;
2215	return rc;
2216	}
2217
2218
2219	/**
2220	* Performs the lazy mapping / updating of a PAE guest PD.
2221	*
2222	* @returns Pointer to the mapping.
2223	* @returns VBox status code.
2224	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2225	* @param iPdpt Which PD entry to map (0..3).
2226	* @param ppPd Where to return the pointer to the mapping. This is
2227	* always set.
2228	*/
2229	int pgmGstLazyMapPaePD(PVMCPU pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2230	{
2231	PVM pVM = pVCpu->CTX_SUFF(pVM);
2232	pgmLock(pVM);
2233
2234	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2235	Assert(pGuestPDPT);
2236	Assert(pGuestPDPT->a[iPdpt].n.u1Present);
2237	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2238	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2239
2240	PPGMPAGE pPage;
2241	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2242	if (RT_SUCCESS(rc))
2243	{
2244	RTRCPTR RCPtr = NIL_RTRCPTR;
2245	RTHCPTR HCPtr = NIL_RTHCPTR;
2246	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2247	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, &HCPtr);
2248	AssertRC(rc);
2249	#endif
2250	if (RT_SUCCESS(rc) && fChanged)
2251	{
2252	RCPtr = (RTRCPTR)(RTRCUINTPTR)(pVM->pgm.s.GCPtrCR3Mapping + (1 + iPdpt) * PAGE_SIZE);
2253	rc = PGMMap(pVM, (RTRCUINTPTR)RCPtr, PGM_PAGE_GET_HCPHYS(pPage), PAGE_SIZE, 0);
2254	}
2255	if (RT_SUCCESS(rc))
2256	{
2257	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = (R3PTRTYPE(PX86PDPAE))HCPtr;
2258	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2259	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = (R0PTRTYPE(PX86PDPAE))HCPtr;
2260	# endif
2261	if (fChanged)
2262	{
2263	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2264	pVCpu->pgm.s.apGstPaePDsRC[iPdpt] = (RCPTRTYPE(PX86PDPAE))RCPtr;
2265	}
2266
2267	*ppPd = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
2268	pgmUnlock(pVM);
2269	return VINF_SUCCESS;
2270	}
2271	}
2272
2273	/* Invalid page or some failure, invalidate the entry. */
2274	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2275	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = 0;
2276	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2277	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = 0;
2278	# endif
2279	pVCpu->pgm.s.apGstPaePDsRC[iPdpt] = 0;
2280
2281	pgmUnlock(pVM);
2282	return rc;
2283	}
2284
2285	#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
2286	#if !defined(IN_RC) && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
2287	/**
2288	* Performs the lazy mapping of the 32-bit guest PD.
2289	*
2290	* @returns VBox status code.
2291	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2292	* @param ppPml4 Where to return the pointer to the mapping. This will
2293	* always be set.
2294	*/
2295	int pgmGstLazyMapPml4(PVMCPU pVCpu, PX86PML4 *ppPml4)
2296	{
2297	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2298	PVM pVM = pVCpu->CTX_SUFF(pVM);
2299	pgmLock(pVM);
2300
2301	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
2302	PPGMPAGE pPage;
2303	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2304	if (RT_SUCCESS(rc))
2305	{
2306	RTHCPTR HCPtrGuestCR3;
2307	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2308	if (RT_SUCCESS(rc))
2309	{
2310	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
2311	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2312	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
2313	# endif
2314	*ppPml4 = (PX86PML4)HCPtrGuestCR3;
2315
2316	pgmUnlock(pVM);
2317	return VINF_SUCCESS;
2318	}
2319	}
2320
2321	pgmUnlock(pVM);
2322	*ppPml4 = NULL;
2323	return rc;
2324	}
2325	#endif
2326
2327
2328	/**
2329	* Gets the PAE PDPEs values cached by the CPU.
2330	*
2331	* @returns VBox status code.
2332	* @param pVCpu The cross context virtual CPU structure.
2333	* @param paPdpes Where to return the four PDPEs. The array
2334	* pointed to must have 4 entries.
2335	*/
2336	VMM_INT_DECL(int) PGMGstGetPaePdpes(PVMCPU pVCpu, PX86PDPE paPdpes)
2337	{
2338	Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2339
2340	paPdpes[0] = pVCpu->pgm.s.aGstPaePdpeRegs[0];
2341	paPdpes[1] = pVCpu->pgm.s.aGstPaePdpeRegs[1];
2342	paPdpes[2] = pVCpu->pgm.s.aGstPaePdpeRegs[2];
2343	paPdpes[3] = pVCpu->pgm.s.aGstPaePdpeRegs[3];
2344	return VINF_SUCCESS;
2345	}
2346
2347
2348	/**
2349	* Sets the PAE PDPEs values cached by the CPU.
2350	*
2351	* @remarks This must be called AFTER PGMUpdateCR3.
2352	*
2353	* @param pVCpu The cross context virtual CPU structure.
2354	* @param paPdpes The four PDPE values. The array pointed to must
2355	* have exactly 4 entries.
2356	*
2357	* @remarks No-long-jump zone!!!
2358	*/
2359	VMM_INT_DECL(void) PGMGstUpdatePaePdpes(PVMCPU pVCpu, PCX86PDPE paPdpes)
2360	{
2361	Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2362
2363	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGstPaePdpeRegs); i++)
2364	{
2365	if (pVCpu->pgm.s.aGstPaePdpeRegs[i].u != paPdpes[i].u)
2366	{
2367	pVCpu->pgm.s.aGstPaePdpeRegs[i] = paPdpes[i];
2368
2369	/* Force lazy remapping if it changed in any way. */
2370	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2371	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2372	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2373	# endif
2374	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
2375	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2376	}
2377	}
2378
2379	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
2380	}
2381
2382
2383	/**
2384	* Gets the current CR3 register value for the shadow memory context.
2385	* @returns CR3 value.
2386	* @param pVCpu The cross context virtual CPU structure.
2387	*/
2388	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2389	{
2390	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2391	AssertPtrReturn(pPoolPage, 0);
2392	return pPoolPage->Core.Key;
2393	}
2394
2395
2396	/**
2397	* Gets the current CR3 register value for the nested memory context.
2398	* @returns CR3 value.
2399	* @param pVCpu The cross context virtual CPU structure.
2400	* @param enmShadowMode The shadow paging mode.
2401	*/
2402	VMMDECL(RTHCPHYS) PGMGetNestedCR3(PVMCPU pVCpu, PGMMODE enmShadowMode)
2403	{
2404	NOREF(enmShadowMode);
2405	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
2406	return pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->Core.Key;
2407	}
2408
2409
2410	/**
2411	* Gets the current CR3 register value for the HC intermediate memory context.
2412	* @returns CR3 value.
2413	* @param pVM The cross context VM structure.
2414	*/
2415	VMMDECL(RTHCPHYS) PGMGetInterHCCR3(PVM pVM)
2416	{
2417	switch (pVM->pgm.s.enmHostMode)
2418	{
2419	case SUPPAGINGMODE_32_BIT:
2420	case SUPPAGINGMODE_32_BIT_GLOBAL:
2421	return pVM->pgm.s.HCPhysInterPD;
2422
2423	case SUPPAGINGMODE_PAE:
2424	case SUPPAGINGMODE_PAE_GLOBAL:
2425	case SUPPAGINGMODE_PAE_NX:
2426	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2427	return pVM->pgm.s.HCPhysInterPaePDPT;
2428
2429	case SUPPAGINGMODE_AMD64:
2430	case SUPPAGINGMODE_AMD64_GLOBAL:
2431	case SUPPAGINGMODE_AMD64_NX:
2432	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2433	return pVM->pgm.s.HCPhysInterPaePDPT;
2434
2435	default:
2436	AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode));
2437	return NIL_RTHCPHYS;
2438	}
2439	}
2440
2441
2442	/**
2443	* Gets the current CR3 register value for the RC intermediate memory context.
2444	* @returns CR3 value.
2445	* @param pVM The cross context VM structure.
2446	* @param pVCpu The cross context virtual CPU structure.
2447	*/
2448	VMMDECL(RTHCPHYS) PGMGetInterRCCR3(PVM pVM, PVMCPU pVCpu)
2449	{
2450	switch (pVCpu->pgm.s.enmShadowMode)
2451	{
2452	case PGMMODE_32_BIT:
2453	return pVM->pgm.s.HCPhysInterPD;
2454
2455	case PGMMODE_PAE:
2456	case PGMMODE_PAE_NX:
2457	return pVM->pgm.s.HCPhysInterPaePDPT;
2458
2459	case PGMMODE_AMD64:
2460	case PGMMODE_AMD64_NX:
2461	return pVM->pgm.s.HCPhysInterPaePML4;
2462
2463	case PGMMODE_NESTED_32BIT:
2464	case PGMMODE_NESTED_PAE:
2465	case PGMMODE_NESTED_AMD64:
2466	case PGMMODE_EPT:
2467	return 0; /* not relevant */
2468
2469	default:
2470	AssertMsgFailed(("enmShadowMode=%d\n", pVCpu->pgm.s.enmShadowMode));
2471	return NIL_RTHCPHYS;
2472	}
2473	}
2474
2475
2476	/**
2477	* Gets the CR3 register value for the 32-Bit intermediate memory context.
2478	* @returns CR3 value.
2479	* @param pVM The cross context VM structure.
2480	*/
2481	VMMDECL(RTHCPHYS) PGMGetInter32BitCR3(PVM pVM)
2482	{
2483	return pVM->pgm.s.HCPhysInterPD;
2484	}
2485
2486
2487	/**
2488	* Gets the CR3 register value for the PAE intermediate memory context.
2489	* @returns CR3 value.
2490	* @param pVM The cross context VM structure.
2491	*/
2492	VMMDECL(RTHCPHYS) PGMGetInterPaeCR3(PVM pVM)
2493	{
2494	return pVM->pgm.s.HCPhysInterPaePDPT;
2495	}
2496
2497
2498	/**
2499	* Gets the CR3 register value for the AMD64 intermediate memory context.
2500	* @returns CR3 value.
2501	* @param pVM The cross context VM structure.
2502	*/
2503	VMMDECL(RTHCPHYS) PGMGetInterAmd64CR3(PVM pVM)
2504	{
2505	return pVM->pgm.s.HCPhysInterPaePML4;
2506	}
2507
2508
2509	/**
2510	* Performs and schedules necessary updates following a CR3 load or reload.
2511	*
2512	* This will normally involve mapping the guest PD or nPDPT
2513	*
2514	* @returns VBox status code.
2515	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2516	* safely be ignored and overridden since the FF will be set too then.
2517	* @param pVCpu The cross context virtual CPU structure.
2518	* @param cr3 The new cr3.
2519	* @param fGlobal Indicates whether this is a global flush or not.
2520	*/
2521	VMMDECL(int) PGMFlushTLB(PVMCPU pVCpu, uint64_t cr3, bool fGlobal)
2522	{
2523	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
2524	PVM pVM = pVCpu->CTX_SUFF(pVM);
2525
2526	VMCPU_ASSERT_EMT(pVCpu);
2527
2528	/*
2529	* Always flag the necessary updates; necessary for hardware acceleration
2530	*/
2531	/** @todo optimize this, it shouldn't always be necessary. */
2532	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2533	if (fGlobal)
2534	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2535	LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVCpu->pgm.s.GCPhysCR3, fGlobal));
2536
2537	/*
2538	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2539	*/
2540	int rc = VINF_SUCCESS;
2541	RTGCPHYS GCPhysCR3;
2542	switch (pVCpu->pgm.s.enmGuestMode)
2543	{
2544	case PGMMODE_PAE:
2545	case PGMMODE_PAE_NX:
2546	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2547	break;
2548	case PGMMODE_AMD64:
2549	case PGMMODE_AMD64_NX:
2550	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2551	break;
2552	default:
2553	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2554	break;
2555	}
2556	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2557
2558	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2559	{
2560	RTGCPHYS GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2561	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2562	rc = PGM_BTH_PFN(MapCR3, pVCpu)(pVCpu, GCPhysCR3);
2563	if (RT_LIKELY(rc == VINF_SUCCESS))
2564	{
2565	if (pgmMapAreMappingsFloating(pVM))
2566	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2567	}
2568	else
2569	{
2570	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2571	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2572	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2573	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2574	if (pgmMapAreMappingsFloating(pVM))
2575	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MONITOR_CR3;
2576	}
2577
2578	if (fGlobal)
2579	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2580	else
2581	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3));
2582	}
2583	else
2584	{
2585	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2586	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2587	if (pPool->cDirtyPages)
2588	{
2589	pgmLock(pVM);
2590	pgmPoolResetDirtyPages(pVM);
2591	pgmUnlock(pVM);
2592	}
2593	# endif
2594	/*
2595	* Check if we have a pending update of the CR3 monitoring.
2596	*/
2597	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2598	{
2599	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2600	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
2601	}
2602	if (fGlobal)
2603	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2604	else
2605	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3));
2606	}
2607
2608	IEMTlbInvalidateAll(pVCpu, false /fVmm/);
2609	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
2610	return rc;
2611	}
2612
2613
2614	/**
2615	* Performs and schedules necessary updates following a CR3 load or reload when
2616	* using nested or extended paging.
2617	*
2618	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2619	* TLB and triggering a SyncCR3.
2620	*
2621	* This will normally involve mapping the guest PD or nPDPT
2622	*
2623	* @returns VBox status code.
2624	* @retval VINF_SUCCESS.
2625	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2626	* paging modes). This can safely be ignored and overridden since the
2627	* FF will be set too then.
2628	* @param pVCpu The cross context virtual CPU structure.
2629	* @param cr3 The new cr3.
2630	*/
2631	VMMDECL(int) PGMUpdateCR3(PVMCPU pVCpu, uint64_t cr3)
2632	{
2633	VMCPU_ASSERT_EMT(pVCpu);
2634	LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysCR3));
2635
2636	/* We assume we're only called in nested paging mode. */
2637	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2638	Assert(!pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2639	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3));
2640
2641	/*
2642	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2643	*/
2644	int rc = VINF_SUCCESS;
2645	RTGCPHYS GCPhysCR3;
2646	switch (pVCpu->pgm.s.enmGuestMode)
2647	{
2648	case PGMMODE_PAE:
2649	case PGMMODE_PAE_NX:
2650	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2651	break;
2652	case PGMMODE_AMD64:
2653	case PGMMODE_AMD64_NX:
2654	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2655	break;
2656	default:
2657	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2658	break;
2659	}
2660	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2661
2662	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2663	{
2664	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2665	rc = PGM_BTH_PFN(MapCR3, pVCpu)(pVCpu, GCPhysCR3);
2666	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2667	}
2668
2669	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2670	return rc;
2671	}
2672
2673
2674	/**
2675	* Synchronize the paging structures.
2676	*
2677	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2678	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2679	* in several places, most importantly whenever the CR3 is loaded.
2680	*
2681	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2682	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2683	* the VMM into guest context.
2684	* @param pVCpu The cross context virtual CPU structure.
2685	* @param cr0 Guest context CR0 register
2686	* @param cr3 Guest context CR3 register
2687	* @param cr4 Guest context CR4 register
2688	* @param fGlobal Including global page directories or not
2689	*/
2690	VMMDECL(int) PGMSyncCR3(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2691	{
2692	int rc;
2693
2694	VMCPU_ASSERT_EMT(pVCpu);
2695
2696	/*
2697	* The pool may have pending stuff and even require a return to ring-3 to
2698	* clear the whole thing.
2699	*/
2700	rc = pgmPoolSyncCR3(pVCpu);
2701	if (rc != VINF_SUCCESS)
2702	return rc;
2703
2704	/*
2705	* We might be called when we shouldn't.
2706	*
2707	* The mode switching will ensure that the PD is resynced after every mode
2708	* switch. So, if we find ourselves here when in protected or real mode
2709	* we can safely clear the FF and return immediately.
2710	*/
2711	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2712	{
2713	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2714	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2715	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2716	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2717	return VINF_SUCCESS;
2718	}
2719
2720	/* If global pages are not supported, then all flushes are global. */
2721	if (!(cr4 & X86_CR4_PGE))
2722	fGlobal = true;
2723	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2724	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2725
2726	/*
2727	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2728	* This should be done before SyncCR3.
2729	*/
2730	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2731	{
2732	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2733
2734	RTGCPHYS GCPhysCR3Old = pVCpu->pgm.s.GCPhysCR3; NOREF(GCPhysCR3Old);
2735	RTGCPHYS GCPhysCR3;
2736	switch (pVCpu->pgm.s.enmGuestMode)
2737	{
2738	case PGMMODE_PAE:
2739	case PGMMODE_PAE_NX:
2740	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2741	break;
2742	case PGMMODE_AMD64:
2743	case PGMMODE_AMD64_NX:
2744	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2745	break;
2746	default:
2747	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2748	break;
2749	}
2750	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2751
2752	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2753	{
2754	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2755	rc = PGM_BTH_PFN(MapCR3, pVCpu)(pVCpu, GCPhysCR3);
2756	}
2757
2758	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2759	if ( rc == VINF_PGM_SYNC_CR3
2760	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2761	{
2762	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2763	#ifdef IN_RING3
2764	rc = pgmPoolSyncCR3(pVCpu);
2765	#else
2766	if (rc == VINF_PGM_SYNC_CR3)
2767	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3Old;
2768	return VINF_PGM_SYNC_CR3;
2769	#endif
2770	}
2771	AssertRCReturn(rc, rc);
2772	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2773	}
2774
2775	/*
2776	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2777	*/
2778	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
2779	rc = PGM_BTH_PFN(SyncCR3, pVCpu)(pVCpu, cr0, cr3, cr4, fGlobal);
2780	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
2781	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2782	if (rc == VINF_SUCCESS)
2783	{
2784	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2785	{
2786	/* Go back to ring 3 if a pgm pool sync is again pending. */
2787	return VINF_PGM_SYNC_CR3;
2788	}
2789
2790	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2791	{
2792	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2793	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2794	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2795	}
2796
2797	/*
2798	* Check if we have a pending update of the CR3 monitoring.
2799	*/
2800	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2801	{
2802	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2803	Assert(!pVCpu->CTX_SUFF(pVM)->pgm.s.fMappingsFixed);
2804	Assert(pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2805	}
2806	}
2807
2808	/*
2809	* Now flush the CR3 (guest context).
2810	*/
2811	if (rc == VINF_SUCCESS)
2812	PGM_INVL_VCPU_TLBS(pVCpu);
2813	return rc;
2814	}
2815
2816
2817	/**
2818	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
2819	*
2820	* @returns VBox status code, with the following informational code for
2821	* VM scheduling.
2822	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
2823	* @retval VINF_PGM_CHANGE_MODE if we're in RC or R0 and the mode changes.
2824	* (I.e. not in R3.)
2825	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
2826	*
2827	* @param pVCpu The cross context virtual CPU structure.
2828	* @param cr0 The new cr0.
2829	* @param cr4 The new cr4.
2830	* @param efer The new extended feature enable register.
2831	*/
2832	VMMDECL(int) PGMChangeMode(PVMCPU pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer)
2833	{
2834	VMCPU_ASSERT_EMT(pVCpu);
2835
2836	/*
2837	* Calc the new guest mode.
2838	*
2839	* Note! We check PG before PE and without requiring PE because of the
2840	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
2841	*/
2842	PGMMODE enmGuestMode;
2843	if (cr0 & X86_CR0_PG)
2844	{
2845	if (!(cr4 & X86_CR4_PAE))
2846	{
2847	bool const fPse = !!(cr4 & X86_CR4_PSE);
2848	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
2849	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
2850	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
2851	enmGuestMode = PGMMODE_32_BIT;
2852	}
2853	else if (!(efer & MSR_K6_EFER_LME))
2854	{
2855	if (!(efer & MSR_K6_EFER_NXE))
2856	enmGuestMode = PGMMODE_PAE;
2857	else
2858	enmGuestMode = PGMMODE_PAE_NX;
2859	}
2860	else
2861	{
2862	if (!(efer & MSR_K6_EFER_NXE))
2863	enmGuestMode = PGMMODE_AMD64;
2864	else
2865	enmGuestMode = PGMMODE_AMD64_NX;
2866	}
2867	}
2868	else if (!(cr0 & X86_CR0_PE))
2869	enmGuestMode = PGMMODE_REAL;
2870	else
2871	enmGuestMode = PGMMODE_PROTECTED;
2872
2873	/*
2874	* Did it change?
2875	*/
2876	if (pVCpu->pgm.s.enmGuestMode == enmGuestMode)
2877	return VINF_SUCCESS;
2878
2879	/* Flush the TLB */
2880	PGM_INVL_VCPU_TLBS(pVCpu);
2881
2882	#ifdef IN_RING3
2883	return PGMR3ChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode);
2884	#else
2885	LogFlow(("PGMChangeMode: returns VINF_PGM_CHANGE_MODE.\n"));
2886	return VINF_PGM_CHANGE_MODE;
2887	#endif
2888	}
2889
2890
2891	/**
2892	* Called by CPUM or REM when CR0.WP changes to 1.
2893	*
2894	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2895	* @thread EMT
2896	*/
2897	VMMDECL(void) PGMCr0WpEnabled(PVMCPU pVCpu)
2898	{
2899	/*
2900	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
2901	*
2902	* Use the counter to judge whether there might be pool pages with active
2903	* hacks in them. If there are, we will be running the risk of messing up
2904	* the guest by allowing it to write to read-only pages. Thus, we have to
2905	* clear the page pool ASAP if there is the slightest chance.
2906	*/
2907	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
2908	{
2909	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
2910
2911	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
2912	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
2913	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2914	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2915	}
2916	}
2917
2918
2919	/**
2920	* Gets the current guest paging mode.
2921	*
2922	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
2923	*
2924	* @returns The current paging mode.
2925	* @param pVCpu The cross context virtual CPU structure.
2926	*/
2927	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
2928	{
2929	return pVCpu->pgm.s.enmGuestMode;
2930	}
2931
2932
2933	/**
2934	* Gets the current shadow paging mode.
2935	*
2936	* @returns The current paging mode.
2937	* @param pVCpu The cross context virtual CPU structure.
2938	*/
2939	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
2940	{
2941	return pVCpu->pgm.s.enmShadowMode;
2942	}
2943
2944
2945	/**
2946	* Gets the current host paging mode.
2947	*
2948	* @returns The current paging mode.
2949	* @param pVM The cross context VM structure.
2950	*/
2951	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
2952	{
2953	switch (pVM->pgm.s.enmHostMode)
2954	{
2955	case SUPPAGINGMODE_32_BIT:
2956	case SUPPAGINGMODE_32_BIT_GLOBAL:
2957	return PGMMODE_32_BIT;
2958
2959	case SUPPAGINGMODE_PAE:
2960	case SUPPAGINGMODE_PAE_GLOBAL:
2961	return PGMMODE_PAE;
2962
2963	case SUPPAGINGMODE_PAE_NX:
2964	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2965	return PGMMODE_PAE_NX;
2966
2967	case SUPPAGINGMODE_AMD64:
2968	case SUPPAGINGMODE_AMD64_GLOBAL:
2969	return PGMMODE_AMD64;
2970
2971	case SUPPAGINGMODE_AMD64_NX:
2972	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2973	return PGMMODE_AMD64_NX;
2974
2975	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
2976	}
2977
2978	return PGMMODE_INVALID;
2979	}
2980
2981
2982	/**
2983	* Get mode name.
2984	*
2985	* @returns read-only name string.
2986	* @param enmMode The mode which name is desired.
2987	*/
2988	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
2989	{
2990	switch (enmMode)
2991	{
2992	case PGMMODE_REAL: return "Real";
2993	case PGMMODE_PROTECTED: return "Protected";
2994	case PGMMODE_32_BIT: return "32-bit";
2995	case PGMMODE_PAE: return "PAE";
2996	case PGMMODE_PAE_NX: return "PAE+NX";
2997	case PGMMODE_AMD64: return "AMD64";
2998	case PGMMODE_AMD64_NX: return "AMD64+NX";
2999	case PGMMODE_NESTED_32BIT: return "Nested-32";
3000	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3001	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3002	case PGMMODE_EPT: return "EPT";
3003	default: return "unknown mode value";
3004	}
3005	}
3006
3007
3008	/**
3009	* Gets the physical address represented in the guest CR3 as PGM sees it.
3010	*
3011	* This is mainly for logging and debugging.
3012	*
3013	* @returns PGM's guest CR3 value.
3014	* @param pVCpu The cross context virtual CPU structure.
3015	*/
3016	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3017	{
3018	return pVCpu->pgm.s.GCPhysCR3;
3019	}
3020
3021
3022
3023	/**
3024	* Notification from CPUM that the EFER.NXE bit has changed.
3025	*
3026	* @param pVCpu The cross context virtual CPU structure of the CPU for
3027	* which EFER changed.
3028	* @param fNxe The new NXE state.
3029	*/
3030	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3031	{
3032	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3033	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3034
3035	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3036	if (fNxe)
3037	{
3038	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3039	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3040	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3041	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3042	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3043	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3044	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3045	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3046	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3047	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3048	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3049
3050	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3051	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3052	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3053	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3054	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3055	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3056	}
3057	else
3058	{
3059	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3060	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3061	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3062	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3063	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3064	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3065	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3066	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3067	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3068	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3069	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3070
3071	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3072	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3073	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3074	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3075	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3076	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3077	}
3078	}
3079
3080
3081	/**
3082	* Check if any pgm pool pages are marked dirty (not monitored)
3083	*
3084	* @returns bool locked/not locked
3085	* @param pVM The cross context VM structure.
3086	*/
3087	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3088	{
3089	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3090	}
3091
3092
3093	/**
3094	* Check if this VCPU currently owns the PGM lock.
3095	*
3096	* @returns bool owner/not owner
3097	* @param pVM The cross context VM structure.
3098	*/
3099	VMMDECL(bool) PGMIsLockOwner(PVM pVM)
3100	{
3101	return PDMCritSectIsOwner(&pVM->pgm.s.CritSectX);
3102	}
3103
3104
3105	/**
3106	* Enable or disable large page usage
3107	*
3108	* @returns VBox status code.
3109	* @param pVM The cross context VM structure.
3110	* @param fUseLargePages Use/not use large pages
3111	*/
3112	VMMDECL(int) PGMSetLargePageUsage(PVM pVM, bool fUseLargePages)
3113	{
3114	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3115
3116	pVM->fUseLargePages = fUseLargePages;
3117	return VINF_SUCCESS;
3118	}
3119
3120
3121	/**
3122	* Acquire the PGM lock.
3123	*
3124	* @returns VBox status code
3125	* @param pVM The cross context VM structure.
3126	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3127	*/
3128	#if (defined(VBOX_STRICT) && defined(IN_RING3)) \|\| defined(DOXYGEN_RUNNING)
3129	int pgmLockDebug(PVM pVM, RT_SRC_POS_DECL)
3130	#else
3131	int pgmLock(PVM pVM)
3132	#endif
3133	{
3134	#if defined(VBOX_STRICT) && defined(IN_RING3)
3135	int rc = PDMCritSectEnterDebug(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3136	#else
3137	int rc = PDMCritSectEnter(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY);
3138	#endif
3139	#if defined(IN_RC) \|\| defined(IN_RING0)
3140	if (rc == VERR_SEM_BUSY)
3141	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_LOCK, 0);
3142	#endif
3143	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
3144	return rc;
3145	}
3146
3147
3148	/**
3149	* Release the PGM lock.
3150	*
3151	* @returns VBox status code
3152	* @param pVM The cross context VM structure.
3153	*/
3154	void pgmUnlock(PVM pVM)
3155	{
3156	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3157	pVM->pgm.s.cDeprecatedPageLocks = 0;
3158	int rc = PDMCritSectLeave(&pVM->pgm.s.CritSectX);
3159	if (rc == VINF_SEM_NESTED)
3160	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3161	}
3162
3163	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3164
3165	/**
3166	* Common worker for pgmRZDynMapGCPageOffInlined and pgmRZDynMapGCPageV2Inlined.
3167	*
3168	* @returns VBox status code.
3169	* @param pVM The cross context VM structure.
3170	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3171	* @param GCPhys The guest physical address of the page to map. The
3172	* offset bits are not ignored.
3173	* @param ppv Where to return the address corresponding to @a GCPhys.
3174	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3175	*/
3176	int pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
3177	{
3178	pgmLock(pVM);
3179
3180	/*
3181	* Convert it to a writable page and it on to the dynamic mapper.
3182	*/
3183	int rc;
3184	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
3185	if (RT_LIKELY(pPage))
3186	{
3187	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3188	if (RT_SUCCESS(rc))
3189	{
3190	void *pv;
3191	rc = pgmRZDynMapHCPageInlined(pVCpu, PGM_PAGE_GET_HCPHYS(pPage), &pv RTLOG_COMMA_SRC_POS_ARGS);
3192	if (RT_SUCCESS(rc))
3193	ppv = (void )((uintptr_t)pv \| ((uintptr_t)GCPhys & PAGE_OFFSET_MASK));
3194	}
3195	else
3196	AssertRC(rc);
3197	}
3198	else
3199	{
3200	AssertMsgFailed(("Invalid physical address %RGp!\n", GCPhys));
3201	rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3202	}
3203
3204	pgmUnlock(pVM);
3205	return rc;
3206	}
3207
3208	#endif /* IN_RC \|\| VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3209	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3210
3211	/** Format handler for PGMPAGE.
3212	* @copydoc FNRTSTRFORMATTYPE */
3213	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3214	const char pszType, void const pvValue,
3215	int cchWidth, int cchPrecision, unsigned fFlags,
3216	void *pvUser)
3217	{
3218	size_t cch;
3219	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3220	if (RT_VALID_PTR(pPage))
3221	{
3222	char szTmp[64+80];
3223
3224	cch = 0;
3225
3226	/* The single char state stuff. */
3227	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3228	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3229
3230	#define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3231	if (IS_PART_INCLUDED(5))
3232	{
3233	static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
3234	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
3235	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_VIRT_STATE(pPage)];
3236	}
3237
3238	/* The type. */
3239	if (IS_PART_INCLUDED(4))
3240	{
3241	szTmp[cch++] = ':';
3242	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3243	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3244	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3245	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3246	}
3247
3248	/* The numbers. */
3249	if (IS_PART_INCLUDED(3))
3250	{
3251	szTmp[cch++] = ':';
3252	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3253	}
3254
3255	if (IS_PART_INCLUDED(2))
3256	{
3257	szTmp[cch++] = ':';
3258	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3259	}
3260
3261	if (IS_PART_INCLUDED(6))
3262	{
3263	szTmp[cch++] = ':';
3264	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3265	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3266	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3267	}
3268	#undef IS_PART_INCLUDED
3269
3270	cch = pfnOutput(pvArgOutput, szTmp, cch);
3271	}
3272	else
3273	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3274	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3275	return cch;
3276	}
3277
3278
3279	/** Format handler for PGMRAMRANGE.
3280	* @copydoc FNRTSTRFORMATTYPE */
3281	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3282	const char pszType, void const pvValue,
3283	int cchWidth, int cchPrecision, unsigned fFlags,
3284	void *pvUser)
3285	{
3286	size_t cch;
3287	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3288	if (VALID_PTR(pRam))
3289	{
3290	char szTmp[80];
3291	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3292	cch = pfnOutput(pvArgOutput, szTmp, cch);
3293	}
3294	else
3295	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3296	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3297	return cch;
3298	}
3299
3300	/** Format type andlers to be registered/deregistered. */
3301	static const struct
3302	{
3303	char szType[24];
3304	PFNRTSTRFORMATTYPE pfnHandler;
3305	} g_aPgmFormatTypes[] =
3306	{
3307	{ "pgmpage", pgmFormatTypeHandlerPage },
3308	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3309	};
3310
3311	#endif /* !IN_R0 \|\| LOG_ENABLED */
3312
3313	/**
3314	* Registers the global string format types.
3315	*
3316	* This should be called at module load time or in some other manner that ensure
3317	* that it's called exactly one time.
3318	*
3319	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3320	*/
3321	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3322	{
3323	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3324	int rc = VINF_SUCCESS;
3325	unsigned i;
3326	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3327	{
3328	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3329	# ifdef IN_RING0
3330	if (rc == VERR_ALREADY_EXISTS)
3331	{
3332	/* in case of cleanup failure in ring-0 */
3333	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3334	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3335	}
3336	# endif
3337	}
3338	if (RT_FAILURE(rc))
3339	while (i-- > 0)
3340	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3341
3342	return rc;
3343	#else
3344	return VINF_SUCCESS;
3345	#endif
3346	}
3347
3348
3349	/**
3350	* Deregisters the global string format types.
3351	*
3352	* This should be called at module unload time or in some other manner that
3353	* ensure that it's called exactly one time.
3354	*/
3355	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
3356	{
3357	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3358	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3359	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3360	#endif
3361	}
3362
3363	#ifdef VBOX_STRICT
3364
3365	/**
3366	* Asserts that there are no mapping conflicts.
3367	*
3368	* @returns Number of conflicts.
3369	* @param pVM The cross context VM structure.
3370	*/
3371	VMMDECL(unsigned) PGMAssertNoMappingConflicts(PVM pVM)
3372	{
3373	unsigned cErrors = 0;
3374
3375	/* Only applies to raw mode -> 1 VPCU */
3376	Assert(pVM->cCpus == 1);
3377	PVMCPU pVCpu = &pVM->aCpus[0];
3378
3379	/*
3380	* Check for mapping conflicts.
3381	*/
3382	for (PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
3383	pMapping;
3384	pMapping = pMapping->CTX_SUFF(pNext))
3385	{
3386	/** @todo This is slow and should be optimized, but since it's just assertions I don't care now. */
3387	for (RTGCPTR GCPtr = pMapping->GCPtr;
3388	GCPtr <= pMapping->GCPtrLast;
3389	GCPtr += PAGE_SIZE)
3390	{
3391	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)GCPtr, NULL, NULL);
3392	if (rc != VERR_PAGE_TABLE_NOT_PRESENT)
3393	{
3394	AssertMsgFailed(("Conflict at %RGv with %s\n", GCPtr, R3STRING(pMapping->pszDesc)));
3395	cErrors++;
3396	break;
3397	}
3398	}
3399	}
3400
3401	return cErrors;
3402	}
3403
3404
3405	/**
3406	* Asserts that everything related to the guest CR3 is correctly shadowed.
3407	*
3408	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
3409	* and assert the correctness of the guest CR3 mapping before asserting that the
3410	* shadow page tables is in sync with the guest page tables.
3411	*
3412	* @returns Number of conflicts.
3413	* @param pVM The cross context VM structure.
3414	* @param pVCpu The cross context virtual CPU structure.
3415	* @param cr3 The current guest CR3 register value.
3416	* @param cr4 The current guest CR4 register value.
3417	*/
3418	VMMDECL(unsigned) PGMAssertCR3(PVM pVM, PVMCPU pVCpu, uint64_t cr3, uint64_t cr4)
3419	{
3420	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
3421	pgmLock(pVM);
3422	unsigned cErrors = PGM_BTH_PFN(AssertCR3, pVCpu)(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3423	pgmUnlock(pVM);
3424	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
3425	return cErrors;
3426	}
3427
3428	#endif /* VBOX_STRICT */
3429

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 73249

Download in other formats: