PGMAll.cpp@ 100765

Last change on this file since 100765 was 99891, checked in by vboxsync, 19 months ago
VMM/PGM: Docs correction for PGMInvalidatePage.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 147.5 KB

Line
1	/* $Id: PGMAll.cpp 99891 2023-05-22 10:55:43Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/selm.h>
37	#include <VBox/vmm/iem.h>
38	#include <VBox/vmm/iom.h>
39	#include <VBox/sup.h>
40	#include <VBox/vmm/mm.h>
41	#include <VBox/vmm/stam.h>
42	#include <VBox/vmm/trpm.h>
43	#include <VBox/vmm/em.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/vmm/hm_vmx.h>
46	#include "PGMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include "PGMInline.h"
49	#include <iprt/assert.h>
50	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
51	# include <iprt/asm-amd64-x86.h>
52	#endif
53	#include <iprt/string.h>
54	#include <VBox/log.h>
55	#include <VBox/param.h>
56	#include <VBox/err.h>
57
58
59	/*********************************************************************************************************************************
60	* Internal Functions *
61	*********************************************************************************************************************************/
62	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
63	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
64	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
65	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
66	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
67	PPGMPTWALKGST pGstWalk);
68	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
69	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
70	PPGMPTWALKGST pGstWalkAll);
71	#endif
72	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
73	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
74
75
76	/*
77	* Second level transation - EPT.
78	*/
79	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
80	# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
81	# include "PGMSlatDefs.h"
82	# include "PGMAllGstSlatEpt.cpp.h"
83	# undef PGM_SLAT_TYPE
84	#endif
85
86
87	/*
88	* Shadow - 32-bit mode
89	*/
90	#define PGM_SHW_TYPE PGM_TYPE_32BIT
91	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
92	#include "PGMAllShw.h"
93
94	/* Guest - real mode */
95	#define PGM_GST_TYPE PGM_TYPE_REAL
96	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
97	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
98	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
99	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
100	#include "PGMGstDefs.h"
101	#include "PGMAllGst.h"
102	#include "PGMAllBth.h"
103	#undef BTH_PGMPOOLKIND_PT_FOR_PT
104	#undef BTH_PGMPOOLKIND_ROOT
105	#undef PGM_BTH_NAME
106	#undef PGM_GST_TYPE
107	#undef PGM_GST_NAME
108
109	/* Guest - protected mode */
110	#define PGM_GST_TYPE PGM_TYPE_PROT
111	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
112	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
113	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
114	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
115	#include "PGMGstDefs.h"
116	#include "PGMAllGst.h"
117	#include "PGMAllBth.h"
118	#undef BTH_PGMPOOLKIND_PT_FOR_PT
119	#undef BTH_PGMPOOLKIND_ROOT
120	#undef PGM_BTH_NAME
121	#undef PGM_GST_TYPE
122	#undef PGM_GST_NAME
123
124	/* Guest - 32-bit mode */
125	#define PGM_GST_TYPE PGM_TYPE_32BIT
126	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
127	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
128	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
129	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
130	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
131	#include "PGMGstDefs.h"
132	#include "PGMAllGst.h"
133	#include "PGMAllBth.h"
134	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
135	#undef BTH_PGMPOOLKIND_PT_FOR_PT
136	#undef BTH_PGMPOOLKIND_ROOT
137	#undef PGM_BTH_NAME
138	#undef PGM_GST_TYPE
139	#undef PGM_GST_NAME
140
141	#undef PGM_SHW_TYPE
142	#undef PGM_SHW_NAME
143
144
145	/*
146	* Shadow - PAE mode
147	*/
148	#define PGM_SHW_TYPE PGM_TYPE_PAE
149	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
150	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
151	#include "PGMAllShw.h"
152
153	/* Guest - real mode */
154	#define PGM_GST_TYPE PGM_TYPE_REAL
155	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
156	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
157	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
158	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
159	#include "PGMGstDefs.h"
160	#include "PGMAllBth.h"
161	#undef BTH_PGMPOOLKIND_PT_FOR_PT
162	#undef BTH_PGMPOOLKIND_ROOT
163	#undef PGM_BTH_NAME
164	#undef PGM_GST_TYPE
165	#undef PGM_GST_NAME
166
167	/* Guest - protected mode */
168	#define PGM_GST_TYPE PGM_TYPE_PROT
169	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
170	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
171	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
172	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
173	#include "PGMGstDefs.h"
174	#include "PGMAllBth.h"
175	#undef BTH_PGMPOOLKIND_PT_FOR_PT
176	#undef BTH_PGMPOOLKIND_ROOT
177	#undef PGM_BTH_NAME
178	#undef PGM_GST_TYPE
179	#undef PGM_GST_NAME
180
181	/* Guest - 32-bit mode */
182	#define PGM_GST_TYPE PGM_TYPE_32BIT
183	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
184	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
185	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
186	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
187	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
188	#include "PGMGstDefs.h"
189	#include "PGMAllBth.h"
190	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
191	#undef BTH_PGMPOOLKIND_PT_FOR_PT
192	#undef BTH_PGMPOOLKIND_ROOT
193	#undef PGM_BTH_NAME
194	#undef PGM_GST_TYPE
195	#undef PGM_GST_NAME
196
197
198	/* Guest - PAE mode */
199	#define PGM_GST_TYPE PGM_TYPE_PAE
200	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
201	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
202	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
203	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
204	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
205	#include "PGMGstDefs.h"
206	#include "PGMAllGst.h"
207	#include "PGMAllBth.h"
208	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
209	#undef BTH_PGMPOOLKIND_PT_FOR_PT
210	#undef BTH_PGMPOOLKIND_ROOT
211	#undef PGM_BTH_NAME
212	#undef PGM_GST_TYPE
213	#undef PGM_GST_NAME
214
215	#undef PGM_SHW_TYPE
216	#undef PGM_SHW_NAME
217
218
219	/*
220	* Shadow - AMD64 mode
221	*/
222	#define PGM_SHW_TYPE PGM_TYPE_AMD64
223	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
224	#include "PGMAllShw.h"
225
226	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
227	/** @todo retire this hack. */
228	#define PGM_GST_TYPE PGM_TYPE_PROT
229	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
230	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
231	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
232	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
233	#include "PGMGstDefs.h"
234	#include "PGMAllBth.h"
235	#undef BTH_PGMPOOLKIND_PT_FOR_PT
236	#undef BTH_PGMPOOLKIND_ROOT
237	#undef PGM_BTH_NAME
238	#undef PGM_GST_TYPE
239	#undef PGM_GST_NAME
240
241	#ifdef VBOX_WITH_64_BITS_GUESTS
242	/* Guest - AMD64 mode */
243	# define PGM_GST_TYPE PGM_TYPE_AMD64
244	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
245	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
246	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
247	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
248	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
249	# include "PGMGstDefs.h"
250	# include "PGMAllGst.h"
251	# include "PGMAllBth.h"
252	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
253	# undef BTH_PGMPOOLKIND_PT_FOR_PT
254	# undef BTH_PGMPOOLKIND_ROOT
255	# undef PGM_BTH_NAME
256	# undef PGM_GST_TYPE
257	# undef PGM_GST_NAME
258	#endif /* VBOX_WITH_64_BITS_GUESTS */
259
260	#undef PGM_SHW_TYPE
261	#undef PGM_SHW_NAME
262
263
264	/*
265	* Shadow - 32-bit nested paging mode.
266	*/
267	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
268	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
269	#include "PGMAllShw.h"
270
271	/* Guest - real mode */
272	#define PGM_GST_TYPE PGM_TYPE_REAL
273	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
274	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
275	#include "PGMGstDefs.h"
276	#include "PGMAllBth.h"
277	#undef PGM_BTH_NAME
278	#undef PGM_GST_TYPE
279	#undef PGM_GST_NAME
280
281	/* Guest - protected mode */
282	#define PGM_GST_TYPE PGM_TYPE_PROT
283	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
284	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
285	#include "PGMGstDefs.h"
286	#include "PGMAllBth.h"
287	#undef PGM_BTH_NAME
288	#undef PGM_GST_TYPE
289	#undef PGM_GST_NAME
290
291	/* Guest - 32-bit mode */
292	#define PGM_GST_TYPE PGM_TYPE_32BIT
293	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
294	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
295	#include "PGMGstDefs.h"
296	#include "PGMAllBth.h"
297	#undef PGM_BTH_NAME
298	#undef PGM_GST_TYPE
299	#undef PGM_GST_NAME
300
301	/* Guest - PAE mode */
302	#define PGM_GST_TYPE PGM_TYPE_PAE
303	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
304	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
305	#include "PGMGstDefs.h"
306	#include "PGMAllBth.h"
307	#undef PGM_BTH_NAME
308	#undef PGM_GST_TYPE
309	#undef PGM_GST_NAME
310
311	#ifdef VBOX_WITH_64_BITS_GUESTS
312	/* Guest - AMD64 mode */
313	# define PGM_GST_TYPE PGM_TYPE_AMD64
314	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
315	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
316	# include "PGMGstDefs.h"
317	# include "PGMAllBth.h"
318	# undef PGM_BTH_NAME
319	# undef PGM_GST_TYPE
320	# undef PGM_GST_NAME
321	#endif /* VBOX_WITH_64_BITS_GUESTS */
322
323	#undef PGM_SHW_TYPE
324	#undef PGM_SHW_NAME
325
326
327	/*
328	* Shadow - PAE nested paging mode.
329	*/
330	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
331	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
332	#include "PGMAllShw.h"
333
334	/* Guest - real mode */
335	#define PGM_GST_TYPE PGM_TYPE_REAL
336	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
337	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
338	#include "PGMGstDefs.h"
339	#include "PGMAllBth.h"
340	#undef PGM_BTH_NAME
341	#undef PGM_GST_TYPE
342	#undef PGM_GST_NAME
343
344	/* Guest - protected mode */
345	#define PGM_GST_TYPE PGM_TYPE_PROT
346	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
347	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
348	#include "PGMGstDefs.h"
349	#include "PGMAllBth.h"
350	#undef PGM_BTH_NAME
351	#undef PGM_GST_TYPE
352	#undef PGM_GST_NAME
353
354	/* Guest - 32-bit mode */
355	#define PGM_GST_TYPE PGM_TYPE_32BIT
356	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
357	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
358	#include "PGMGstDefs.h"
359	#include "PGMAllBth.h"
360	#undef PGM_BTH_NAME
361	#undef PGM_GST_TYPE
362	#undef PGM_GST_NAME
363
364	/* Guest - PAE mode */
365	#define PGM_GST_TYPE PGM_TYPE_PAE
366	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
367	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
368	#include "PGMGstDefs.h"
369	#include "PGMAllBth.h"
370	#undef PGM_BTH_NAME
371	#undef PGM_GST_TYPE
372	#undef PGM_GST_NAME
373
374	#ifdef VBOX_WITH_64_BITS_GUESTS
375	/* Guest - AMD64 mode */
376	# define PGM_GST_TYPE PGM_TYPE_AMD64
377	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
378	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
379	# include "PGMGstDefs.h"
380	# include "PGMAllBth.h"
381	# undef PGM_BTH_NAME
382	# undef PGM_GST_TYPE
383	# undef PGM_GST_NAME
384	#endif /* VBOX_WITH_64_BITS_GUESTS */
385
386	#undef PGM_SHW_TYPE
387	#undef PGM_SHW_NAME
388
389
390	/*
391	* Shadow - AMD64 nested paging mode.
392	*/
393	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
394	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
395	#include "PGMAllShw.h"
396
397	/* Guest - real mode */
398	#define PGM_GST_TYPE PGM_TYPE_REAL
399	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
400	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
401	#include "PGMGstDefs.h"
402	#include "PGMAllBth.h"
403	#undef PGM_BTH_NAME
404	#undef PGM_GST_TYPE
405	#undef PGM_GST_NAME
406
407	/* Guest - protected mode */
408	#define PGM_GST_TYPE PGM_TYPE_PROT
409	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
410	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
411	#include "PGMGstDefs.h"
412	#include "PGMAllBth.h"
413	#undef PGM_BTH_NAME
414	#undef PGM_GST_TYPE
415	#undef PGM_GST_NAME
416
417	/* Guest - 32-bit mode */
418	#define PGM_GST_TYPE PGM_TYPE_32BIT
419	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
420	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
421	#include "PGMGstDefs.h"
422	#include "PGMAllBth.h"
423	#undef PGM_BTH_NAME
424	#undef PGM_GST_TYPE
425	#undef PGM_GST_NAME
426
427	/* Guest - PAE mode */
428	#define PGM_GST_TYPE PGM_TYPE_PAE
429	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
430	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
431	#include "PGMGstDefs.h"
432	#include "PGMAllBth.h"
433	#undef PGM_BTH_NAME
434	#undef PGM_GST_TYPE
435	#undef PGM_GST_NAME
436
437	#ifdef VBOX_WITH_64_BITS_GUESTS
438	/* Guest - AMD64 mode */
439	# define PGM_GST_TYPE PGM_TYPE_AMD64
440	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
441	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
442	# include "PGMGstDefs.h"
443	# include "PGMAllBth.h"
444	# undef PGM_BTH_NAME
445	# undef PGM_GST_TYPE
446	# undef PGM_GST_NAME
447	#endif /* VBOX_WITH_64_BITS_GUESTS */
448
449	#undef PGM_SHW_TYPE
450	#undef PGM_SHW_NAME
451
452
453	/*
454	* Shadow - EPT.
455	*/
456	#define PGM_SHW_TYPE PGM_TYPE_EPT
457	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
458	#include "PGMAllShw.h"
459
460	/* Guest - real mode */
461	#define PGM_GST_TYPE PGM_TYPE_REAL
462	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
463	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
464	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
465	#include "PGMGstDefs.h"
466	#include "PGMAllBth.h"
467	#undef BTH_PGMPOOLKIND_PT_FOR_PT
468	#undef PGM_BTH_NAME
469	#undef PGM_GST_TYPE
470	#undef PGM_GST_NAME
471
472	/* Guest - protected mode */
473	#define PGM_GST_TYPE PGM_TYPE_PROT
474	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
475	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
476	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
477	#include "PGMGstDefs.h"
478	#include "PGMAllBth.h"
479	#undef BTH_PGMPOOLKIND_PT_FOR_PT
480	#undef PGM_BTH_NAME
481	#undef PGM_GST_TYPE
482	#undef PGM_GST_NAME
483
484	/* Guest - 32-bit mode */
485	#define PGM_GST_TYPE PGM_TYPE_32BIT
486	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
487	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
488	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
489	#include "PGMGstDefs.h"
490	#include "PGMAllBth.h"
491	#undef BTH_PGMPOOLKIND_PT_FOR_PT
492	#undef PGM_BTH_NAME
493	#undef PGM_GST_TYPE
494	#undef PGM_GST_NAME
495
496	/* Guest - PAE mode */
497	#define PGM_GST_TYPE PGM_TYPE_PAE
498	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
499	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
500	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
501	#include "PGMGstDefs.h"
502	#include "PGMAllBth.h"
503	#undef BTH_PGMPOOLKIND_PT_FOR_PT
504	#undef PGM_BTH_NAME
505	#undef PGM_GST_TYPE
506	#undef PGM_GST_NAME
507
508	#ifdef VBOX_WITH_64_BITS_GUESTS
509	/* Guest - AMD64 mode */
510	# define PGM_GST_TYPE PGM_TYPE_AMD64
511	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
512	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
513	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
514	# include "PGMGstDefs.h"
515	# include "PGMAllBth.h"
516	# undef BTH_PGMPOOLKIND_PT_FOR_PT
517	# undef PGM_BTH_NAME
518	# undef PGM_GST_TYPE
519	# undef PGM_GST_NAME
520	#endif /* VBOX_WITH_64_BITS_GUESTS */
521
522	#undef PGM_SHW_TYPE
523	#undef PGM_SHW_NAME
524
525
526	/*
527	* Shadow - NEM / None.
528	*/
529	#define PGM_SHW_TYPE PGM_TYPE_NONE
530	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
531	#include "PGMAllShw.h"
532
533	/* Guest - real mode */
534	#define PGM_GST_TYPE PGM_TYPE_REAL
535	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
536	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
537	#include "PGMGstDefs.h"
538	#include "PGMAllBth.h"
539	#undef PGM_BTH_NAME
540	#undef PGM_GST_TYPE
541	#undef PGM_GST_NAME
542
543	/* Guest - protected mode */
544	#define PGM_GST_TYPE PGM_TYPE_PROT
545	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
546	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
547	#include "PGMGstDefs.h"
548	#include "PGMAllBth.h"
549	#undef PGM_BTH_NAME
550	#undef PGM_GST_TYPE
551	#undef PGM_GST_NAME
552
553	/* Guest - 32-bit mode */
554	#define PGM_GST_TYPE PGM_TYPE_32BIT
555	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
556	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
557	#include "PGMGstDefs.h"
558	#include "PGMAllBth.h"
559	#undef PGM_BTH_NAME
560	#undef PGM_GST_TYPE
561	#undef PGM_GST_NAME
562
563	/* Guest - PAE mode */
564	#define PGM_GST_TYPE PGM_TYPE_PAE
565	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
566	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
567	#include "PGMGstDefs.h"
568	#include "PGMAllBth.h"
569	#undef PGM_BTH_NAME
570	#undef PGM_GST_TYPE
571	#undef PGM_GST_NAME
572
573	#ifdef VBOX_WITH_64_BITS_GUESTS
574	/* Guest - AMD64 mode */
575	# define PGM_GST_TYPE PGM_TYPE_AMD64
576	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
577	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
578	# include "PGMGstDefs.h"
579	# include "PGMAllBth.h"
580	# undef PGM_BTH_NAME
581	# undef PGM_GST_TYPE
582	# undef PGM_GST_NAME
583	#endif /* VBOX_WITH_64_BITS_GUESTS */
584
585	#undef PGM_SHW_TYPE
586	#undef PGM_SHW_NAME
587
588
589
590	/**
591	* Guest mode data array.
592	*/
593	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
594	{
595	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
596	{
597	PGM_TYPE_REAL,
598	PGM_GST_NAME_REAL(GetPage),
599	PGM_GST_NAME_REAL(ModifyPage),
600	PGM_GST_NAME_REAL(Enter),
601	PGM_GST_NAME_REAL(Exit),
602	#ifdef IN_RING3
603	PGM_GST_NAME_REAL(Relocate),
604	#endif
605	},
606	{
607	PGM_TYPE_PROT,
608	PGM_GST_NAME_PROT(GetPage),
609	PGM_GST_NAME_PROT(ModifyPage),
610	PGM_GST_NAME_PROT(Enter),
611	PGM_GST_NAME_PROT(Exit),
612	#ifdef IN_RING3
613	PGM_GST_NAME_PROT(Relocate),
614	#endif
615	},
616	{
617	PGM_TYPE_32BIT,
618	PGM_GST_NAME_32BIT(GetPage),
619	PGM_GST_NAME_32BIT(ModifyPage),
620	PGM_GST_NAME_32BIT(Enter),
621	PGM_GST_NAME_32BIT(Exit),
622	#ifdef IN_RING3
623	PGM_GST_NAME_32BIT(Relocate),
624	#endif
625	},
626	{
627	PGM_TYPE_PAE,
628	PGM_GST_NAME_PAE(GetPage),
629	PGM_GST_NAME_PAE(ModifyPage),
630	PGM_GST_NAME_PAE(Enter),
631	PGM_GST_NAME_PAE(Exit),
632	#ifdef IN_RING3
633	PGM_GST_NAME_PAE(Relocate),
634	#endif
635	},
636	#ifdef VBOX_WITH_64_BITS_GUESTS
637	{
638	PGM_TYPE_AMD64,
639	PGM_GST_NAME_AMD64(GetPage),
640	PGM_GST_NAME_AMD64(ModifyPage),
641	PGM_GST_NAME_AMD64(Enter),
642	PGM_GST_NAME_AMD64(Exit),
643	# ifdef IN_RING3
644	PGM_GST_NAME_AMD64(Relocate),
645	# endif
646	},
647	#endif
648	};
649
650
651	/**
652	* The shadow mode data array.
653	*/
654	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
655	{
656	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
657	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
658	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
659	{
660	PGM_TYPE_32BIT,
661	PGM_SHW_NAME_32BIT(GetPage),
662	PGM_SHW_NAME_32BIT(ModifyPage),
663	PGM_SHW_NAME_32BIT(Enter),
664	PGM_SHW_NAME_32BIT(Exit),
665	#ifdef IN_RING3
666	PGM_SHW_NAME_32BIT(Relocate),
667	#endif
668	},
669	{
670	PGM_TYPE_PAE,
671	PGM_SHW_NAME_PAE(GetPage),
672	PGM_SHW_NAME_PAE(ModifyPage),
673	PGM_SHW_NAME_PAE(Enter),
674	PGM_SHW_NAME_PAE(Exit),
675	#ifdef IN_RING3
676	PGM_SHW_NAME_PAE(Relocate),
677	#endif
678	},
679	{
680	PGM_TYPE_AMD64,
681	PGM_SHW_NAME_AMD64(GetPage),
682	PGM_SHW_NAME_AMD64(ModifyPage),
683	PGM_SHW_NAME_AMD64(Enter),
684	PGM_SHW_NAME_AMD64(Exit),
685	#ifdef IN_RING3
686	PGM_SHW_NAME_AMD64(Relocate),
687	#endif
688	},
689	{
690	PGM_TYPE_NESTED_32BIT,
691	PGM_SHW_NAME_NESTED_32BIT(GetPage),
692	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
693	PGM_SHW_NAME_NESTED_32BIT(Enter),
694	PGM_SHW_NAME_NESTED_32BIT(Exit),
695	#ifdef IN_RING3
696	PGM_SHW_NAME_NESTED_32BIT(Relocate),
697	#endif
698	},
699	{
700	PGM_TYPE_NESTED_PAE,
701	PGM_SHW_NAME_NESTED_PAE(GetPage),
702	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
703	PGM_SHW_NAME_NESTED_PAE(Enter),
704	PGM_SHW_NAME_NESTED_PAE(Exit),
705	#ifdef IN_RING3
706	PGM_SHW_NAME_NESTED_PAE(Relocate),
707	#endif
708	},
709	{
710	PGM_TYPE_NESTED_AMD64,
711	PGM_SHW_NAME_NESTED_AMD64(GetPage),
712	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
713	PGM_SHW_NAME_NESTED_AMD64(Enter),
714	PGM_SHW_NAME_NESTED_AMD64(Exit),
715	#ifdef IN_RING3
716	PGM_SHW_NAME_NESTED_AMD64(Relocate),
717	#endif
718	},
719	{
720	PGM_TYPE_EPT,
721	PGM_SHW_NAME_EPT(GetPage),
722	PGM_SHW_NAME_EPT(ModifyPage),
723	PGM_SHW_NAME_EPT(Enter),
724	PGM_SHW_NAME_EPT(Exit),
725	#ifdef IN_RING3
726	PGM_SHW_NAME_EPT(Relocate),
727	#endif
728	},
729	{
730	PGM_TYPE_NONE,
731	PGM_SHW_NAME_NONE(GetPage),
732	PGM_SHW_NAME_NONE(ModifyPage),
733	PGM_SHW_NAME_NONE(Enter),
734	PGM_SHW_NAME_NONE(Exit),
735	#ifdef IN_RING3
736	PGM_SHW_NAME_NONE(Relocate),
737	#endif
738	},
739	};
740
741
742	/**
743	* The guest+shadow mode data array.
744	*/
745	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
746	{
747	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
748	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
749	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
750	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
751
752	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
753	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
754	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
755	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
756
757	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
758	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
759	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
760	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
761
762	#elif defined(IN_RING3) && defined(VBOX_STRICT)
763	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
764	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
765	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
766
767	#else
768	# error "Misconfig."
769	#endif
770
771	/* 32-bit shadow paging mode: */
772	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
773	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
774	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
775	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
776	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
777	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
778	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
780	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
781	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
782	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
783
784	/* PAE shadow paging mode: */
785	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
786	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
787	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
788	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
789	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
790	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
791	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
793	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
796
797	/* AMD64 shadow paging mode: */
798	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
799	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
800	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
801	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
802	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
803	#ifdef VBOX_WITH_64_BITS_GUESTS
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
805	#else
806	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
807	#endif
808	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
810	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
811	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
812	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
813
814	/* 32-bit nested paging mode: */
815	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
816	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
817	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
818	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
820	#ifdef VBOX_WITH_64_BITS_GUESTS
821	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
822	#else
823	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
824	#endif
825	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
827	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
828	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
829	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
830
831	/* PAE nested paging mode: */
832	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
833	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
834	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
835	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
837	#ifdef VBOX_WITH_64_BITS_GUESTS
838	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
839	#else
840	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
841	#endif
842	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
844	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
845	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
846	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
847
848	/* AMD64 nested paging mode: */
849	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
850	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
851	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
852	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
854	#ifdef VBOX_WITH_64_BITS_GUESTS
855	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
856	#else
857	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
858	#endif
859	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
861	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
862	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
863	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
864
865	/* EPT nested paging mode: */
866	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
867	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
868	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
869	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
870	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
871	#ifdef VBOX_WITH_64_BITS_GUESTS
872	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
873	#else
874	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
875	#endif
876	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
877	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
878	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
879	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
880	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
881
882	/* NONE / NEM: */
883	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
884	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
885	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
886	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
887	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
888	#ifdef VBOX_WITH_64_BITS_GUESTS
889	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
890	#else
891	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
892	#endif
893	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
894	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
895	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
896	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
897	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
898
899
900	#undef PGMMODEDATABTH_ENTRY
901	#undef PGMMODEDATABTH_NULL_ENTRY
902	};
903
904
905	/** Mask array used by pgmGetCr3MaskForMode.
906	* X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
907	static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
908	{
909	/* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK,
910	/* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK,
911	/* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK,
912	/* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK,
913	/* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK,
914	/* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK,
915	/* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
916	/* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK,
917	/* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK,
918	/* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK,
919	/* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
920	/* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK,
921	/* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK,
922	};
923
924
925	/**
926	* Gets the physical address mask for CR3 in the given paging mode.
927	*
928	* The mask is for eliminating flags and other stuff in CR3/EPTP when
929	* extracting the physical address. It is not for validating whether there are
930	* reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed
931	* it to PGM checked for reserved bits, including reserved physical address
932	* bits.
933	*
934	* @returns The CR3 mask.
935	* @param enmMode The paging mode.
936	* @param enmSlatMode The second-level address translation mode.
937	*/
938	DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
939	{
940	if (enmSlatMode == PGMSLAT_DIRECT)
941	{
942	Assert(enmMode != PGMMODE_EPT);
943	return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
944	}
945	Assert(enmSlatMode == PGMSLAT_EPT);
946	return X86_CR3_EPT_PAGE_MASK;
947	}
948
949
950	/**
951	* Gets the masked CR3 value according to the current guest paging mode.
952	*
953	* See disclaimer in pgmGetCr3MaskForMode.
954	*
955	* @returns The masked PGM CR3 value.
956	* @param pVCpu The cross context virtual CPU structure.
957	* @param uCr3 The raw guest CR3 value.
958	*/
959	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
960	{
961	uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
962	RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
963	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
964	return GCPhysCR3;
965	}
966
967
968	#ifdef IN_RING0
969	/**
970	* #PF Handler.
971	*
972	* @returns VBox status code (appropriate for trap handling and GC return).
973	* @param pVCpu The cross context virtual CPU structure.
974	* @param uErr The trap error code.
975	* @param pCtx Pointer to the register context for the CPU.
976	* @param pvFault The fault address.
977	*/
978	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
979	{
980	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
981
982	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
983	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
984	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
985
986
987	# ifdef VBOX_WITH_STATISTICS
988	/*
989	* Error code stats.
990	*/
991	if (uErr & X86_TRAP_PF_US)
992	{
993	if (!(uErr & X86_TRAP_PF_P))
994	{
995	if (uErr & X86_TRAP_PF_RW)
996	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
997	else
998	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
999	}
1000	else if (uErr & X86_TRAP_PF_RW)
1001	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1002	else if (uErr & X86_TRAP_PF_RSVD)
1003	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1004	else if (uErr & X86_TRAP_PF_ID)
1005	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1006	else
1007	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1008	}
1009	else
1010	{ /* Supervisor */
1011	if (!(uErr & X86_TRAP_PF_P))
1012	{
1013	if (uErr & X86_TRAP_PF_RW)
1014	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1015	else
1016	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1017	}
1018	else if (uErr & X86_TRAP_PF_RW)
1019	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1020	else if (uErr & X86_TRAP_PF_ID)
1021	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1022	else if (uErr & X86_TRAP_PF_RSVD)
1023	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1024	}
1025	# endif /* VBOX_WITH_STATISTICS */
1026
1027	/*
1028	* Call the worker.
1029	*/
1030	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1031	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1032	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
1033	bool fLockTaken = false;
1034	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
1035	if (fLockTaken)
1036	{
1037	PGM_LOCK_ASSERT_OWNER(pVM);
1038	PGM_UNLOCK(pVM);
1039	}
1040	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
1041
1042	/*
1043	* Return code tweaks.
1044	*/
1045	if (rc != VINF_SUCCESS)
1046	{
1047	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1048	rc = VINF_SUCCESS;
1049
1050	/* Note: hack alert for difficult to reproduce problem. */
1051	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1052	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1053	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1054	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1055	{
1056	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
1057	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
1058	rc = VINF_SUCCESS;
1059	}
1060	}
1061
1062	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
1063	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1064	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1065	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1066	return rc;
1067	}
1068	#endif /* IN_RING0 */
1069
1070
1071	/**
1072	* Prefetch a page
1073	*
1074	* Typically used to sync commonly used pages before entering raw mode
1075	* after a CR3 reload.
1076	*
1077	* @returns VBox status code suitable for scheduling.
1078	* @retval VINF_SUCCESS on success.
1079	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1080	* @param pVCpu The cross context virtual CPU structure.
1081	* @param GCPtrPage Page to invalidate.
1082	*/
1083	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1084	{
1085	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1086
1087	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1088	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1089	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1090	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1091
1092	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1093	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1094	return rc;
1095	}
1096
1097
1098	/**
1099	* Emulation of the invlpg instruction (HC only actually).
1100	*
1101	* @returns Strict VBox status code, special care required.
1102	* @retval VINF_PGM_SYNC_CR3 - handled.
1103	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1104	*
1105	* @param pVCpu The cross context virtual CPU structure.
1106	* @param GCPtrPage Page to invalidate.
1107	*
1108	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1109	* safe, but there could be edge cases!
1110	*
1111	* @todo Flush page or page directory only if necessary!
1112	* @todo VBOXSTRICTRC
1113	*/
1114	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1115	{
1116	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1117	int rc;
1118	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1119
1120	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1121
1122	/*
1123	* Call paging mode specific worker.
1124	*/
1125	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1126	PGM_LOCK_VOID(pVM);
1127
1128	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1129	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1130	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1131	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1132
1133	PGM_UNLOCK(pVM);
1134	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1135
1136	/* Ignore all irrelevant error codes. */
1137	if ( rc == VERR_PAGE_NOT_PRESENT
1138	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1139	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1140	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1141	rc = VINF_SUCCESS;
1142
1143	return rc;
1144	}
1145
1146
1147	/**
1148	* Executes an instruction using the interpreter.
1149	*
1150	* @returns VBox status code (appropriate for trap handling and GC return).
1151	* @param pVCpu The cross context virtual CPU structure.
1152	* @param pvFault Fault address.
1153	*/
1154	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
1155	{
1156	RT_NOREF(pvFault);
1157	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
1158	if (rc == VERR_EM_INTERPRETER)
1159	rc = VINF_EM_RAW_EMULATE_INSTR;
1160	if (rc != VINF_SUCCESS)
1161	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1162	return rc;
1163	}
1164
1165
1166	/**
1167	* Gets effective page information (from the VMM page directory).
1168	*
1169	* @returns VBox status code.
1170	* @param pVCpu The cross context virtual CPU structure.
1171	* @param GCPtr Guest Context virtual address of the page.
1172	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1173	* @param pHCPhys Where to store the HC physical address of the page.
1174	* This is page aligned.
1175	* @remark You should use PGMMapGetPage() for pages in a mapping.
1176	*/
1177	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1178	{
1179	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1180	PGM_LOCK_VOID(pVM);
1181
1182	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1183	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1184	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1185	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1186
1187	PGM_UNLOCK(pVM);
1188	return rc;
1189	}
1190
1191
1192	/**
1193	* Modify page flags for a range of pages in the shadow context.
1194	*
1195	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1196	*
1197	* @returns VBox status code.
1198	* @param pVCpu The cross context virtual CPU structure.
1199	* @param GCPtr Virtual address of the first page in the range.
1200	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1201	* @param fMask The AND mask - page flags X86_PTE_*.
1202	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1203	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1204	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1205	*/
1206	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1207	{
1208	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1209	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1210
1211	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1212
1213	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1214	PGM_LOCK_VOID(pVM);
1215
1216	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1217	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1218	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1219	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1220
1221	PGM_UNLOCK(pVM);
1222	return rc;
1223	}
1224
1225
1226	/**
1227	* Changing the page flags for a single page in the shadow page tables so as to
1228	* make it read-only.
1229	*
1230	* @returns VBox status code.
1231	* @param pVCpu The cross context virtual CPU structure.
1232	* @param GCPtr Virtual address of the first page in the range.
1233	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1234	*/
1235	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1236	{
1237	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1238	}
1239
1240
1241	/**
1242	* Changing the page flags for a single page in the shadow page tables so as to
1243	* make it writable.
1244	*
1245	* The call must know with 101% certainty that the guest page tables maps this
1246	* as writable too. This function will deal shared, zero and write monitored
1247	* pages.
1248	*
1249	* @returns VBox status code.
1250	* @param pVCpu The cross context virtual CPU structure.
1251	* @param GCPtr Virtual address of the first page in the range.
1252	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1253	*/
1254	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1255	{
1256	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1257	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1258	return VINF_SUCCESS;
1259	}
1260
1261
1262	/**
1263	* Changing the page flags for a single page in the shadow page tables so as to
1264	* make it not present.
1265	*
1266	* @returns VBox status code.
1267	* @param pVCpu The cross context virtual CPU structure.
1268	* @param GCPtr Virtual address of the first page in the range.
1269	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1270	*/
1271	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1272	{
1273	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1274	}
1275
1276
1277	/**
1278	* Changing the page flags for a single page in the shadow page tables so as to
1279	* make it supervisor and writable.
1280	*
1281	* This if for dealing with CR0.WP=0 and readonly user pages.
1282	*
1283	* @returns VBox status code.
1284	* @param pVCpu The cross context virtual CPU structure.
1285	* @param GCPtr Virtual address of the first page in the range.
1286	* @param fBigPage Whether or not this is a big page. If it is, we have to
1287	* change the shadow PDE as well. If it isn't, the caller
1288	* has checked that the shadow PDE doesn't need changing.
1289	* We ASSUME 4KB pages backing the big page here!
1290	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1291	*/
1292	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1293	{
1294	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1295	if (rc == VINF_SUCCESS && fBigPage)
1296	{
1297	/* this is a bit ugly... */
1298	switch (pVCpu->pgm.s.enmShadowMode)
1299	{
1300	case PGMMODE_32_BIT:
1301	{
1302	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1303	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1304	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1305	pPde->u \|= X86_PDE_RW;
1306	Log(("-> PDE=%#llx (32)\n", pPde->u));
1307	break;
1308	}
1309	case PGMMODE_PAE:
1310	case PGMMODE_PAE_NX:
1311	{
1312	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1313	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1314	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1315	pPde->u \|= X86_PDE_RW;
1316	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1317	break;
1318	}
1319	default:
1320	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1321	}
1322	}
1323	return rc;
1324	}
1325
1326
1327	/**
1328	* Gets the shadow page directory for the specified address, PAE.
1329	*
1330	* @returns Pointer to the shadow PD.
1331	* @param pVCpu The cross context virtual CPU structure.
1332	* @param GCPtr The address.
1333	* @param uGstPdpe Guest PDPT entry. Valid.
1334	* @param ppPD Receives address of page directory
1335	*/
1336	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1337	{
1338	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1339	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1340	PPGMPOOLPAGE pShwPage;
1341	int rc;
1342	PGM_LOCK_ASSERT_OWNER(pVM);
1343
1344
1345	/* Allocate page directory if not present. */
1346	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1347	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1348	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1349	X86PGPAEUINT const uPdpe = pPdpe->u;
1350	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1351	{
1352	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1353	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1354	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1355
1356	pgmPoolCacheUsed(pPool, pShwPage);
1357
1358	/* Update the entry if necessary. */
1359	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1360	if (uPdpeNew == uPdpe)
1361	{ /* likely */ }
1362	else
1363	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1364	}
1365	else
1366	{
1367	RTGCPTR64 GCPdPt;
1368	PGMPOOLKIND enmKind;
1369	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1370	{
1371	/* AMD-V nested paging or real/protected mode without paging. */
1372	GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1373	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1374	}
1375	else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1376	{
1377	if (uGstPdpe & X86_PDPE_P)
1378	{
1379	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1380	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1381	}
1382	else
1383	{
1384	/* PD not present; guest must reload CR3 to change it.
1385	* No need to monitor anything in this case. */
1386	/** @todo r=bird: WTF is hit?!? */
1387	/Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? /
1388	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1389	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1390	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1391	}
1392	}
1393	else
1394	{
1395	GCPdPt = CPUMGetGuestCR3(pVCpu);
1396	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1397	}
1398
1399	/* Create a reference back to the PDPT by using the index in its shadow page. */
1400	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1401	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1402	&pShwPage);
1403	AssertRCReturn(rc, rc);
1404
1405	/* Hook it up. */
1406	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1407	}
1408	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1409
1410	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1411	return VINF_SUCCESS;
1412	}
1413
1414
1415	/**
1416	* Gets the pointer to the shadow page directory entry for an address, PAE.
1417	*
1418	* @returns Pointer to the PDE.
1419	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1420	* @param GCPtr The address.
1421	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1422	*/
1423	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1424	{
1425	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1426	PGM_LOCK_ASSERT_OWNER(pVM);
1427
1428	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1429	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1430	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1431	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1432	if (!(uPdpe & X86_PDPE_P))
1433	{
1434	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1435	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1436	}
1437	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1438
1439	/* Fetch the pgm pool shadow descriptor. */
1440	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1441	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1442
1443	*ppShwPde = pShwPde;
1444	return VINF_SUCCESS;
1445	}
1446
1447
1448	/**
1449	* Syncs the SHADOW page directory pointer for the specified address.
1450	*
1451	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1452	*
1453	* The caller is responsible for making sure the guest has a valid PD before
1454	* calling this function.
1455	*
1456	* @returns VBox status code.
1457	* @param pVCpu The cross context virtual CPU structure.
1458	* @param GCPtr The address.
1459	* @param uGstPml4e Guest PML4 entry (valid).
1460	* @param uGstPdpe Guest PDPT entry (valid).
1461	* @param ppPD Receives address of page directory
1462	*/
1463	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1464	{
1465	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1466	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1467	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1468	int rc;
1469
1470	PGM_LOCK_ASSERT_OWNER(pVM);
1471
1472	/*
1473	* PML4.
1474	*/
1475	PPGMPOOLPAGE pShwPage;
1476	{
1477	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1478	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1479	X86PGPAEUINT const uPml4e = pPml4e->u;
1480
1481	/* Allocate page directory pointer table if not present. */
1482	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1483	{
1484	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1485	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1486
1487	pgmPoolCacheUsed(pPool, pShwPage);
1488
1489	/* Update the entry if needed. */
1490	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1491	\| (uPml4e & PGM_PML4_FLAGS);
1492	if (uPml4e == uPml4eNew)
1493	{ /* likely */ }
1494	else
1495	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1496	}
1497	else
1498	{
1499	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1500
1501	RTGCPTR64 GCPml4;
1502	PGMPOOLKIND enmKind;
1503	if (fNestedPagingOrNoGstPaging)
1504	{
1505	/* AMD-V nested paging or real/protected mode without paging */
1506	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1507	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1508	}
1509	else
1510	{
1511	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1512	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1513	}
1514
1515	/* Create a reference back to the PDPT by using the index in its shadow page. */
1516	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1517	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1518	&pShwPage);
1519	AssertRCReturn(rc, rc);
1520
1521	/* Hook it up. */
1522	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1523	\| (uPml4e & PGM_PML4_FLAGS));
1524	}
1525	}
1526
1527	/*
1528	* PDPT.
1529	*/
1530	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1531	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1532	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1533	X86PGPAEUINT const uPdpe = pPdpe->u;
1534
1535	/* Allocate page directory if not present. */
1536	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1537	{
1538	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1539	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1540
1541	pgmPoolCacheUsed(pPool, pShwPage);
1542
1543	/* Update the entry if needed. */
1544	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1545	\| (uPdpe & PGM_PDPT_FLAGS);
1546	if (uPdpe == uPdpeNew)
1547	{ /* likely */ }
1548	else
1549	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1550	}
1551	else
1552	{
1553	RTGCPTR64 GCPdPt;
1554	PGMPOOLKIND enmKind;
1555	if (fNestedPagingOrNoGstPaging)
1556	{
1557	/* AMD-V nested paging or real/protected mode without paging */
1558	GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1559	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1560	}
1561	else
1562	{
1563	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1564	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1565	}
1566
1567	/* Create a reference back to the PDPT by using the index in its shadow page. */
1568	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1569	pShwPage->idx, iPdPt, false /fLockPage/,
1570	&pShwPage);
1571	AssertRCReturn(rc, rc);
1572
1573	/* Hook it up. */
1574	ASMAtomicWriteU64(&pPdpe->u,
1575	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1576	}
1577
1578	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1579	return VINF_SUCCESS;
1580	}
1581
1582
1583	/**
1584	* Gets the SHADOW page directory pointer for the specified address (long mode).
1585	*
1586	* @returns VBox status code.
1587	* @param pVCpu The cross context virtual CPU structure.
1588	* @param GCPtr The address.
1589	* @param ppPml4e Receives the address of the page map level 4 entry.
1590	* @param ppPdpt Receives the address of the page directory pointer table.
1591	* @param ppPD Receives the address of the page directory.
1592	*/
1593	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1594	{
1595	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1596	PGM_LOCK_ASSERT_OWNER(pVM);
1597
1598	/*
1599	* PML4
1600	*/
1601	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1602	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1603	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1604	if (ppPml4e)
1605	*ppPml4e = (PX86PML4E)pPml4e;
1606	X86PGPAEUINT const uPml4e = pPml4e->u;
1607	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1608	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1609	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1610
1611	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1612	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1613	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1614
1615	/*
1616	* PDPT
1617	*/
1618	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1619	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1620	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1621	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1622	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1623
1624	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1625	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1626
1627	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1628	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1629	return VINF_SUCCESS;
1630	}
1631
1632
1633	/**
1634	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1635	* backing pages in case the PDPT or PML4 entry is missing.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param GCPtr The address.
1640	* @param ppPdpt Receives address of pdpt
1641	* @param ppPD Receives address of page directory
1642	*/
1643	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1644	{
1645	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1646	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1647	int rc;
1648
1649	Assert(pVM->pgm.s.fNestedPaging);
1650	PGM_LOCK_ASSERT_OWNER(pVM);
1651
1652	/*
1653	* PML4 level.
1654	*/
1655	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1656	Assert(pPml4);
1657
1658	/* Allocate page directory pointer table if not present. */
1659	PPGMPOOLPAGE pShwPage;
1660	{
1661	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1662	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1663	EPTPML4E Pml4e;
1664	Pml4e.u = pPml4e->u;
1665	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1666	{
1667	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1668	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1669	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1670	&pShwPage);
1671	AssertRCReturn(rc, rc);
1672
1673	/* Hook up the new PDPT now. */
1674	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1675	}
1676	else
1677	{
1678	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1679	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1680
1681	pgmPoolCacheUsed(pPool, pShwPage);
1682
1683	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1684	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1685	{ }
1686	else
1687	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1688	}
1689	}
1690
1691	/*
1692	* PDPT level.
1693	*/
1694	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1695	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1696	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1697
1698	if (ppPdpt)
1699	*ppPdpt = pPdpt;
1700
1701	/* Allocate page directory if not present. */
1702	EPTPDPTE Pdpe;
1703	Pdpe.u = pPdpe->u;
1704	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1705	{
1706	RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1707	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1708	pShwPage->idx, iPdPt, false /fLockPage/,
1709	&pShwPage);
1710	AssertRCReturn(rc, rc);
1711
1712	/* Hook up the new PD now. */
1713	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1714	}
1715	else
1716	{
1717	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1718	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1719
1720	pgmPoolCacheUsed(pPool, pShwPage);
1721
1722	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1723	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1724	{ }
1725	else
1726	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1727	}
1728
1729	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1730	return VINF_SUCCESS;
1731	}
1732
1733
1734	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1735	/**
1736	* Syncs the SHADOW nested-guest page directory pointer for the specified address.
1737	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1738	*
1739	* @returns VBox status code.
1740	* @param pVCpu The cross context virtual CPU structure.
1741	* @param GCPhysNested The nested-guest physical address.
1742	* @param ppPdpt Where to store the PDPT. Optional, can be NULL.
1743	* @param ppPD Where to store the PD. Optional, can be NULL.
1744	* @param pGstWalkAll The guest walk info.
1745	*/
1746	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
1747	PPGMPTWALKGST pGstWalkAll)
1748	{
1749	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1750	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1751	int rc;
1752
1753	PPGMPOOLPAGE pShwPage;
1754	Assert(pVM->pgm.s.fNestedPaging);
1755	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
1756	PGM_LOCK_ASSERT_OWNER(pVM);
1757
1758	/*
1759	* PML4 level.
1760	*/
1761	{
1762	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1763	Assert(pPml4);
1764
1765	/* Allocate page directory pointer table if not present. */
1766	{
1767	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
1768	const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1769	PEPTPML4E pPml4e = &pPml4->a[iPml4e];
1770
1771	if (!(pPml4e->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1772	{
1773	RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
1774	rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
1775	PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /fLockPage/,
1776	&pShwPage);
1777	AssertRCReturn(rc, rc);
1778
1779	/* Hook up the new PDPT now. */
1780	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1781	}
1782	else
1783	{
1784	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1785	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1786
1787	pgmPoolCacheUsed(pPool, pShwPage);
1788
1789	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1790	if (pPml4e->u != (pShwPage->Core.Key \| fShwFlags))
1791	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1792	}
1793	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1794	Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
1795	}
1796	}
1797
1798	/*
1799	* PDPT level.
1800	*/
1801	{
1802	AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
1803
1804	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1805	if (ppPdpt)
1806	*ppPdpt = pPdpt;
1807
1808	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
1809	const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1810	PEPTPDPTE pPdpte = &pPdpt->a[iPdPte];
1811
1812	if (!(pPdpte->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1813	{
1814	RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
1815	rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1816	pShwPage->idx, iPdPte, false /fLockPage/, &pShwPage);
1817	AssertRCReturn(rc, rc);
1818
1819	/* Hook up the new PD now. */
1820	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1821	}
1822	else
1823	{
1824	pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
1825	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1826
1827	pgmPoolCacheUsed(pPool, pShwPage);
1828
1829	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1830	if (pPdpte->u != (pShwPage->Core.Key \| fShwFlags))
1831	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1832	}
1833	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1834	Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
1835
1836	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1837	}
1838
1839	return VINF_SUCCESS;
1840	}
1841	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1842
1843
1844	#ifdef IN_RING0
1845	/**
1846	* Synchronizes a range of nested page table entries.
1847	*
1848	* The caller must own the PGM lock.
1849	*
1850	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1851	* @param GCPhys Where to start.
1852	* @param cPages How many pages which entries should be synced.
1853	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1854	* host paging mode for AMD-V).
1855	*/
1856	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1857	{
1858	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1859
1860	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1861	int rc;
1862	switch (enmShwPagingMode)
1863	{
1864	case PGMMODE_32_BIT:
1865	{
1866	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1867	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1868	break;
1869	}
1870
1871	case PGMMODE_PAE:
1872	case PGMMODE_PAE_NX:
1873	{
1874	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1875	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1876	break;
1877	}
1878
1879	case PGMMODE_AMD64:
1880	case PGMMODE_AMD64_NX:
1881	{
1882	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1883	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1884	break;
1885	}
1886
1887	case PGMMODE_EPT:
1888	{
1889	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1890	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1891	break;
1892	}
1893
1894	default:
1895	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1896	}
1897	return rc;
1898	}
1899	#endif /* IN_RING0 */
1900
1901
1902	/**
1903	* Gets effective Guest OS page information.
1904	*
1905	* When GCPtr is in a big page, the function will return as if it was a normal
1906	* 4KB page. If the need for distinguishing between big and normal page becomes
1907	* necessary at a later point, a PGMGstGetPage() will be created for that
1908	* purpose.
1909	*
1910	* @returns VBox status code.
1911	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1912	* @param GCPtr Guest Context virtual address of the page.
1913	* @param pWalk Where to store the page walk information.
1914	*/
1915	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
1916	{
1917	VMCPU_ASSERT_EMT(pVCpu);
1918	Assert(pWalk);
1919	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1920	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1921	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1922	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
1923	}
1924
1925
1926	/**
1927	* Maps the guest CR3.
1928	*
1929	* @returns VBox status code.
1930	* @param pVCpu The cross context virtual CPU structure.
1931	* @param GCPhysCr3 The guest CR3 value.
1932	* @param pHCPtrGuestCr3 Where to store the mapped memory.
1933	*/
1934	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
1935	{
1936	/** @todo this needs some reworking wrt. locking? */
1937	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1938	PGM_LOCK_VOID(pVM);
1939	PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
1940	AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
1941
1942	RTHCPTR HCPtrGuestCr3;
1943	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
1944	PGM_UNLOCK(pVM);
1945
1946	*pHCPtrGuestCr3 = HCPtrGuestCr3;
1947	return rc;
1948	}
1949
1950
1951	/**
1952	* Unmaps the guest CR3.
1953	*
1954	* @returns VBox status code.
1955	* @param pVCpu The cross context virtual CPU structure.
1956	*/
1957	DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
1958	{
1959	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1960	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1961	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
1962	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
1963	}
1964
1965
1966	/**
1967	* Performs a guest page table walk.
1968	*
1969	* The guest should be in paged protect mode or long mode when making a call to
1970	* this function.
1971	*
1972	* @returns VBox status code.
1973	* @retval VINF_SUCCESS on success.
1974	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1975	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1976	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1977	*
1978	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1979	* @param GCPtr The guest virtual address to walk by.
1980	* @param pWalk Where to return the walk result. This is valid for some
1981	* error codes as well.
1982	* @param pGstWalk The guest mode specific page walk information.
1983	*/
1984	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
1985	{
1986	VMCPU_ASSERT_EMT(pVCpu);
1987	switch (pVCpu->pgm.s.enmGuestMode)
1988	{
1989	case PGMMODE_32_BIT:
1990	pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1991	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
1992
1993	case PGMMODE_PAE:
1994	case PGMMODE_PAE_NX:
1995	pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1996	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
1997
1998	case PGMMODE_AMD64:
1999	case PGMMODE_AMD64_NX:
2000	pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
2001	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
2002
2003	case PGMMODE_REAL:
2004	case PGMMODE_PROTECTED:
2005	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2006	return VERR_PGM_NOT_USED_IN_MODE;
2007
2008	case PGMMODE_EPT:
2009	case PGMMODE_NESTED_32BIT:
2010	case PGMMODE_NESTED_PAE:
2011	case PGMMODE_NESTED_AMD64:
2012	default:
2013	AssertFailed();
2014	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2015	return VERR_PGM_NOT_USED_IN_MODE;
2016	}
2017	}
2018
2019
2020	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2021	/**
2022	* Performs a guest second-level address translation (SLAT).
2023	*
2024	* @returns VBox status code.
2025	* @retval VINF_SUCCESS on success.
2026	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2027	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2028	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2029	*
2030	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2031	* @param GCPhysNested The nested-guest physical address being translated.
2032	* @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the
2033	* cause for this translation.
2034	* @param GCPtrNested The nested-guest virtual address that initiated the
2035	* SLAT. If none, pass 0 (and not NIL_RTGCPTR).
2036	* @param pWalk Where to return the walk result. This is updated for
2037	* all error codes other than
2038	* VERR_PGM_NOT_USED_IN_MODE.
2039	* @param pGstWalk Where to store the second-level paging-mode specific
2040	* walk info.
2041	*/
2042	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
2043	PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2044	{
2045	/* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
2046	Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
2047	&& pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
2048	AssertPtr(pWalk);
2049	AssertPtr(pGstWalk);
2050	switch (pVCpu->pgm.s.enmGuestSlatMode)
2051	{
2052	case PGMSLAT_EPT:
2053	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
2054	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
2055
2056	default:
2057	AssertFailed();
2058	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2059	return VERR_PGM_NOT_USED_IN_MODE;
2060	}
2061	}
2062	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
2063
2064
2065	/**
2066	* Tries to continue the previous walk.
2067	*
2068	* @note Requires the caller to hold the PGM lock from the first
2069	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
2070	* we cannot use the pointers.
2071	*
2072	* @returns VBox status code.
2073	* @retval VINF_SUCCESS on success.
2074	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2075	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2076	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2077	*
2078	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2079	* @param GCPtr The guest virtual address to walk by.
2080	* @param pWalk Pointer to the previous walk result and where to return
2081	* the result of this walk. This is valid for some error
2082	* codes as well.
2083	* @param pGstWalk The guest-mode specific walk information.
2084	*/
2085	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2086	{
2087	/*
2088	* We can only handle successfully walks.
2089	* We also limit ourselves to the next page.
2090	*/
2091	if ( pWalk->fSucceeded
2092	&& GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
2093	{
2094	Assert(pWalk->uLevel == 0);
2095	if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2096	{
2097	/*
2098	* AMD64
2099	*/
2100	if (!pWalk->fGigantPage && !pWalk->fBigPage)
2101	{
2102	/*
2103	* We fall back to full walk if the PDE table changes, if any
2104	* reserved bits are set, or if the effective page access changes.
2105	*/
2106	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
2107	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
2108	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
2109	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
2110
2111	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
2112	{
2113	if (pGstWalk->u.Amd64.pPte)
2114	{
2115	X86PTEPAE Pte;
2116	Pte.u = pGstWalk->u.Amd64.pPte[1].u;
2117	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2118	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2119	{
2120	pWalk->GCPtr = GCPtr;
2121	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2122	pGstWalk->u.Amd64.Pte.u = Pte.u;
2123	pGstWalk->u.Amd64.pPte++;
2124	return VINF_SUCCESS;
2125	}
2126	}
2127	}
2128	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
2129	{
2130	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2131	if (pGstWalk->u.Amd64.pPde)
2132	{
2133	X86PDEPAE Pde;
2134	Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2135	if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2136	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2137	{
2138	/* Get the new PTE and check out the first entry. */
2139	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2140	&pGstWalk->u.Amd64.pPt);
2141	if (RT_SUCCESS(rc))
2142	{
2143	pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2144	X86PTEPAE Pte;
2145	Pte.u = pGstWalk->u.Amd64.pPte->u;
2146	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2147	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2148	{
2149	pWalk->GCPtr = GCPtr;
2150	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2151	pGstWalk->u.Amd64.Pte.u = Pte.u;
2152	pGstWalk->u.Amd64.Pde.u = Pde.u;
2153	pGstWalk->u.Amd64.pPde++;
2154	return VINF_SUCCESS;
2155	}
2156	}
2157	}
2158	}
2159	}
2160	}
2161	else if (!pWalk->fGigantPage)
2162	{
2163	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2164	{
2165	pWalk->GCPtr = GCPtr;
2166	pWalk->GCPhys += GUEST_PAGE_SIZE;
2167	return VINF_SUCCESS;
2168	}
2169	}
2170	else
2171	{
2172	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2173	{
2174	pWalk->GCPtr = GCPtr;
2175	pWalk->GCPhys += GUEST_PAGE_SIZE;
2176	return VINF_SUCCESS;
2177	}
2178	}
2179	}
2180	}
2181	/* Case we don't handle. Do full walk. */
2182	return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2183	}
2184
2185
2186	/**
2187	* Modify page flags for a range of pages in the guest's tables
2188	*
2189	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2190	*
2191	* @returns VBox status code.
2192	* @param pVCpu The cross context virtual CPU structure.
2193	* @param GCPtr Virtual address of the first page in the range.
2194	* @param cb Size (in bytes) of the range to apply the modification to.
2195	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2196	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2197	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2198	*/
2199	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2200	{
2201	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2202	VMCPU_ASSERT_EMT(pVCpu);
2203
2204	/*
2205	* Validate input.
2206	*/
2207	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2208	Assert(cb);
2209
2210	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2211
2212	/*
2213	* Adjust input.
2214	*/
2215	cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2216	cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2217	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2218
2219	/*
2220	* Call worker.
2221	*/
2222	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2223	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2224	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2225	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2226
2227	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2228	return rc;
2229	}
2230
2231
2232	/**
2233	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2234	*
2235	* @returns @c true if the PDPE is valid, @c false otherwise.
2236	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2237	* @param paPaePdpes The PAE PDPEs to validate.
2238	*
2239	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2240	*/
2241	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2242	{
2243	Assert(paPaePdpes);
2244	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2245	{
2246	X86PDPE const PaePdpe = paPaePdpes[i];
2247	if ( !(PaePdpe.u & X86_PDPE_P)
2248	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2249	{ /* likely */ }
2250	else
2251	return false;
2252	}
2253	return true;
2254	}
2255
2256
2257	/**
2258	* Performs the lazy mapping of the 32-bit guest PD.
2259	*
2260	* @returns VBox status code.
2261	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2262	* @param ppPd Where to return the pointer to the mapping. This is
2263	* always set.
2264	*/
2265	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2266	{
2267	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2268	PGM_LOCK_VOID(pVM);
2269
2270	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2271
2272	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2273	PPGMPAGE pPage;
2274	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2275	if (RT_SUCCESS(rc))
2276	{
2277	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2278	if (RT_SUCCESS(rc))
2279	{
2280	# ifdef IN_RING3
2281	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2282	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2283	# else
2284	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2285	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2286	# endif
2287	PGM_UNLOCK(pVM);
2288	return VINF_SUCCESS;
2289	}
2290	AssertRC(rc);
2291	}
2292	PGM_UNLOCK(pVM);
2293
2294	*ppPd = NULL;
2295	return rc;
2296	}
2297
2298
2299	/**
2300	* Performs the lazy mapping of the PAE guest PDPT.
2301	*
2302	* @returns VBox status code.
2303	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2304	* @param ppPdpt Where to return the pointer to the mapping. This is
2305	* always set.
2306	*/
2307	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2308	{
2309	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2310	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2311	PGM_LOCK_VOID(pVM);
2312
2313	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2314	PPGMPAGE pPage;
2315	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2316	if (RT_SUCCESS(rc))
2317	{
2318	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2319	if (RT_SUCCESS(rc))
2320	{
2321	# ifdef IN_RING3
2322	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2323	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2324	# else
2325	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2326	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2327	# endif
2328	PGM_UNLOCK(pVM);
2329	return VINF_SUCCESS;
2330	}
2331	AssertRC(rc);
2332	}
2333
2334	PGM_UNLOCK(pVM);
2335	*ppPdpt = NULL;
2336	return rc;
2337	}
2338
2339
2340	/**
2341	* Performs the lazy mapping / updating of a PAE guest PD.
2342	*
2343	* @returns Pointer to the mapping.
2344	* @returns VBox status code.
2345	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2346	* @param iPdpt Which PD entry to map (0..3).
2347	* @param ppPd Where to return the pointer to the mapping. This is
2348	* always set.
2349	*/
2350	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2351	{
2352	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2353	PGM_LOCK_VOID(pVM);
2354
2355	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2356	Assert(pGuestPDPT);
2357	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2358	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2359	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2360
2361	PPGMPAGE pPage;
2362	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2363	if (RT_SUCCESS(rc))
2364	{
2365	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2366	AssertRC(rc);
2367	if (RT_SUCCESS(rc))
2368	{
2369	# ifdef IN_RING3
2370	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2371	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2372	# else
2373	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2374	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2375	# endif
2376	if (fChanged)
2377	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2378	PGM_UNLOCK(pVM);
2379	return VINF_SUCCESS;
2380	}
2381	}
2382
2383	/* Invalid page or some failure, invalidate the entry. */
2384	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2385	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2386	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2387
2388	PGM_UNLOCK(pVM);
2389	return rc;
2390	}
2391
2392
2393	/**
2394	* Performs the lazy mapping of the 32-bit guest PD.
2395	*
2396	* @returns VBox status code.
2397	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2398	* @param ppPml4 Where to return the pointer to the mapping. This will
2399	* always be set.
2400	*/
2401	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2402	{
2403	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2404	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2405	PGM_LOCK_VOID(pVM);
2406
2407	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2408	PPGMPAGE pPage;
2409	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2410	if (RT_SUCCESS(rc))
2411	{
2412	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2413	if (RT_SUCCESS(rc))
2414	{
2415	# ifdef IN_RING3
2416	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2417	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2418	# else
2419	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2420	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2421	# endif
2422	PGM_UNLOCK(pVM);
2423	return VINF_SUCCESS;
2424	}
2425	}
2426
2427	PGM_UNLOCK(pVM);
2428	*ppPml4 = NULL;
2429	return rc;
2430	}
2431
2432
2433	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2434	/**
2435	* Performs the lazy mapping of the guest PML4 table when using EPT paging.
2436	*
2437	* @returns VBox status code.
2438	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2439	* @param ppEptPml4 Where to return the pointer to the mapping. This will
2440	* always be set.
2441	*/
2442	int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2443	{
2444	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2445	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2446	PGM_LOCK_VOID(pVM);
2447
2448	RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2449	PPGMPAGE pPage;
2450	int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2451	if (RT_SUCCESS(rc))
2452	{
2453	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2454	if (RT_SUCCESS(rc))
2455	{
2456	# ifdef IN_RING3
2457	pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2458	pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2459	# else
2460	pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2461	pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2462	# endif
2463	PGM_UNLOCK(pVM);
2464	return VINF_SUCCESS;
2465	}
2466	}
2467
2468	PGM_UNLOCK(pVM);
2469	*ppEptPml4 = NULL;
2470	return rc;
2471	}
2472	#endif
2473
2474
2475	/**
2476	* Gets the current CR3 register value for the shadow memory context.
2477	* @returns CR3 value.
2478	* @param pVCpu The cross context virtual CPU structure.
2479	*/
2480	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2481	{
2482	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2483	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2484	return pPoolPage->Core.Key;
2485	}
2486
2487
2488	/**
2489	* Forces lazy remapping of the guest's PAE page-directory structures.
2490	*
2491	* @param pVCpu The cross context virtual CPU structure.
2492	*/
2493	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2494	{
2495	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2496	{
2497	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2498	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2499	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2500	}
2501	}
2502
2503
2504	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2505	/**
2506	* Performs second-level address translation for the given CR3 and updates the
2507	* nested-guest CR3 when successful.
2508	*
2509	* @returns VBox status code.
2510	* @param pVCpu The cross context virtual CPU structure.
2511	* @param uCr3 The masked nested-guest CR3 value.
2512	* @param pGCPhysCR3 Where to store the translated CR3.
2513	*
2514	* @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2515	* mindful of this in code that's hyper sensitive to the order of
2516	* operations.
2517	*/
2518	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2519	{
2520	if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2521	{
2522	PGMPTWALK Walk;
2523	PGMPTWALKGST GstWalk;
2524	int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk, &GstWalk);
2525	if (RT_SUCCESS(rc))
2526	{
2527	/* Update nested-guest CR3. */
2528	pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2529
2530	/* Pass back the translated result. */
2531	*pGCPhysCr3 = Walk.GCPhys;
2532	return VINF_SUCCESS;
2533	}
2534
2535	/* Translation failed. */
2536	*pGCPhysCr3 = NIL_RTGCPHYS;
2537	return rc;
2538	}
2539
2540	/*
2541	* If the nested-guest CR3 has not changed, then the previously
2542	* translated CR3 result (i.e. GCPhysCR3) is passed back.
2543	*/
2544	*pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2545	return VINF_SUCCESS;
2546	}
2547	#endif
2548
2549
2550	/**
2551	* Performs and schedules necessary updates following a CR3 load or reload.
2552	*
2553	* This will normally involve mapping the guest PD or nPDPT
2554	*
2555	* @returns VBox status code.
2556	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2557	* safely be ignored and overridden since the FF will be set too then.
2558	* @param pVCpu The cross context virtual CPU structure.
2559	* @param cr3 The new cr3.
2560	* @param fGlobal Indicates whether this is a global flush or not.
2561	*/
2562	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2563	{
2564	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2565	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2566
2567	VMCPU_ASSERT_EMT(pVCpu);
2568
2569	/*
2570	* Always flag the necessary updates; necessary for hardware acceleration
2571	*/
2572	/** @todo optimize this, it shouldn't always be necessary. */
2573	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2574	if (fGlobal)
2575	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2576
2577	/*
2578	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2579	*/
2580	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2581	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2582	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2583	if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2584	&& PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2585	{
2586	RTGCPHYS GCPhysOut;
2587	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2588	if (RT_SUCCESS(rc))
2589	GCPhysCR3 = GCPhysOut;
2590	else
2591	{
2592	/* CR3 SLAT translation failed but we try to pretend it
2593	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2594	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2595	int const rc2 = pgmGstUnmapCr3(pVCpu);
2596	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2597	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2598	return rc2;
2599	}
2600	}
2601	#endif
2602
2603	LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2604	int rc = VINF_SUCCESS;
2605	if (GCPhysOldCR3 != GCPhysCR3)
2606	{
2607	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2608	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2609	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2610
2611	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2612	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2613	if (RT_LIKELY(rc == VINF_SUCCESS))
2614	{ }
2615	else
2616	{
2617	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2618	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2619	pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2620	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2621	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2622	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2623	}
2624
2625	if (fGlobal)
2626	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2627	else
2628	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2629	}
2630	else
2631	{
2632	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2633	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2634	if (pPool->cDirtyPages)
2635	{
2636	PGM_LOCK_VOID(pVM);
2637	pgmPoolResetDirtyPages(pVM);
2638	PGM_UNLOCK(pVM);
2639	}
2640	#endif
2641	if (fGlobal)
2642	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2643	else
2644	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2645
2646	/*
2647	* Flush PAE PDPTEs.
2648	*/
2649	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2650	pgmGstFlushPaePdpes(pVCpu);
2651	}
2652
2653	IEMTlbInvalidateAll(pVCpu);
2654	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2655	return rc;
2656	}
2657
2658
2659	/**
2660	* Performs and schedules necessary updates following a CR3 load or reload when
2661	* using nested or extended paging.
2662	*
2663	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2664	* TLB and triggering a SyncCR3.
2665	*
2666	* This will normally involve mapping the guest PD or nPDPT
2667	*
2668	* @returns VBox status code.
2669	* @retval VINF_SUCCESS.
2670	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2671	* paging modes). This can safely be ignored and overridden since the
2672	* FF will be set too then.
2673	* @param pVCpu The cross context virtual CPU structure.
2674	* @param cr3 The new CR3.
2675	*/
2676	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2677	{
2678	VMCPU_ASSERT_EMT(pVCpu);
2679
2680	/* We assume we're only called in nested paging mode. */
2681	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2682
2683	/*
2684	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2685	*/
2686	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2687	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2688	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2689	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2690	{
2691	RTGCPHYS GCPhysOut;
2692	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2693	if (RT_SUCCESS(rc))
2694	GCPhysCR3 = GCPhysOut;
2695	else
2696	{
2697	/* CR3 SLAT translation failed but we try to pretend it
2698	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2699	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2700	int const rc2 = pgmGstUnmapCr3(pVCpu);
2701	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2702	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2703	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2704	return rc2;
2705	}
2706	}
2707	#endif
2708
2709	LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2710	int rc = VINF_SUCCESS;
2711	if (GCPhysOldCR3 != GCPhysCR3)
2712	{
2713	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2714	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2715	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2716
2717	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2718	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2719
2720	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2721	}
2722	/*
2723	* Flush PAE PDPTEs.
2724	*/
2725	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2726	pgmGstFlushPaePdpes(pVCpu);
2727
2728	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2729	return rc;
2730	}
2731
2732
2733	/**
2734	* Synchronize the paging structures.
2735	*
2736	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2737	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2738	* in several places, most importantly whenever the CR3 is loaded.
2739	*
2740	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2741	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2742	* the VMM into guest context.
2743	* @param pVCpu The cross context virtual CPU structure.
2744	* @param cr0 Guest context CR0 register
2745	* @param cr3 Guest context CR3 register
2746	* @param cr4 Guest context CR4 register
2747	* @param fGlobal Including global page directories or not
2748	*/
2749	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2750	{
2751	int rc;
2752
2753	VMCPU_ASSERT_EMT(pVCpu);
2754
2755	/*
2756	* The pool may have pending stuff and even require a return to ring-3 to
2757	* clear the whole thing.
2758	*/
2759	rc = pgmPoolSyncCR3(pVCpu);
2760	if (rc != VINF_SUCCESS)
2761	return rc;
2762
2763	/*
2764	* We might be called when we shouldn't.
2765	*
2766	* The mode switching will ensure that the PD is resynced after every mode
2767	* switch. So, if we find ourselves here when in protected or real mode
2768	* we can safely clear the FF and return immediately.
2769	*/
2770	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2771	{
2772	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2773	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2775	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2776	return VINF_SUCCESS;
2777	}
2778
2779	/* If global pages are not supported, then all flushes are global. */
2780	if (!(cr4 & X86_CR4_PGE))
2781	fGlobal = true;
2782	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2783	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2784
2785	/*
2786	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2787	* This should be done before SyncCR3.
2788	*/
2789	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2790	{
2791	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2792
2793	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2794	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2795	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2796	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2797	{
2798	RTGCPHYS GCPhysOut;
2799	int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2800	if (RT_SUCCESS(rc2))
2801	GCPhysCR3 = GCPhysOut;
2802	else
2803	{
2804	/* CR3 SLAT translation failed but we try to pretend it
2805	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2806	AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2));
2807	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2808	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2809	return rc2;
2810	}
2811	}
2812	#endif
2813	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2814	if (GCPhysOldCR3 != GCPhysCR3)
2815	{
2816	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2817	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2818	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2819	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2820	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2821	}
2822
2823	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2824	if ( rc == VINF_PGM_SYNC_CR3
2825	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2826	{
2827	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2828	#ifdef IN_RING3
2829	rc = pgmPoolSyncCR3(pVCpu);
2830	#else
2831	if (rc == VINF_PGM_SYNC_CR3)
2832	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2833	return VINF_PGM_SYNC_CR3;
2834	#endif
2835	}
2836	AssertRCReturn(rc, rc);
2837	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2838	}
2839
2840	/*
2841	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2842	*/
2843	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2844
2845	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2846	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2847	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2848	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2849
2850	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2851	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2852	if (rc == VINF_SUCCESS)
2853	{
2854	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2855	{
2856	/* Go back to ring 3 if a pgm pool sync is again pending. */
2857	return VINF_PGM_SYNC_CR3;
2858	}
2859
2860	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2861	{
2862	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2863	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2864	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2865	}
2866	}
2867
2868	/*
2869	* Now flush the CR3 (guest context).
2870	*/
2871	if (rc == VINF_SUCCESS)
2872	PGM_INVL_VCPU_TLBS(pVCpu);
2873	return rc;
2874	}
2875
2876
2877	/**
2878	* Maps all the PAE PDPE entries.
2879	*
2880	* @returns VBox status code.
2881	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2882	* @param paPaePdpes The new PAE PDPE values.
2883	*
2884	* @remarks This function may be invoked during the process of changing the guest
2885	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
2886	* reflect PAE paging just yet.
2887	*/
2888	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2889	{
2890	Assert(paPaePdpes);
2891	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2892	{
2893	X86PDPE const PaePdpe = paPaePdpes[i];
2894
2895	/*
2896	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
2897	* are deferred.[1] Also, different situations require different handling of invalid
2898	* PDPE entries. Here we assume the caller has already validated or doesn't require
2899	* validation of the PDPEs.
2900	*
2901	* In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
2902	* validated by the VMX transition.
2903	*
2904	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
2905	*/
2906	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
2907	{
2908	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2909	RTHCPTR HCPtr;
2910
2911	RTGCPHYS GCPhys;
2912	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2913	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2914	{
2915	PGMPTWALK Walk;
2916	PGMPTWALKGST GstWalk;
2917	RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
2918	int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid /, 0 / GCPtrNested */,
2919	&Walk, &GstWalk);
2920	if (RT_SUCCESS(rc))
2921	GCPhys = Walk.GCPhys;
2922	else
2923	{
2924	/*
2925	* Second-level address translation of the PAE PDPE has failed but we must -NOT-
2926	* abort and return a failure now. This is because we're called from a Mov CRx
2927	* instruction (or similar operation). Let's just pretend success but flag that
2928	* we need to map this PDPE lazily later.
2929	*
2930	* See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
2931	* See Intel spec. 28.3.1 "EPT Overview".
2932	*/
2933	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2934	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2935	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2936	continue;
2937	}
2938	}
2939	else
2940	#endif
2941	{
2942	GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
2943	}
2944
2945	PGM_LOCK_VOID(pVM);
2946	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2947	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
2948	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
2949	PGM_UNLOCK(pVM);
2950	if (RT_SUCCESS(rc))
2951	{
2952	#ifdef IN_RING3
2953	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
2954	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
2955	#else
2956	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
2957	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
2958	#endif
2959	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
2960	continue;
2961	}
2962	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
2963	}
2964	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2965	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2966	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2967	}
2968	return VINF_SUCCESS;
2969	}
2970
2971
2972	/**
2973	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
2974	*
2975	* @returns VBox status code.
2976	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2977	* @param cr3 The guest CR3 value.
2978	*
2979	* @remarks This function may be invoked during the process of changing the guest
2980	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
2981	* PAE paging just yet.
2982	*/
2983	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
2984	{
2985	/*
2986	* Read the page-directory-pointer table (PDPT) at CR3.
2987	*/
2988	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
2989	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2990
2991	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2992	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2993	{
2994	RTGCPHYS GCPhysOut;
2995	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2996	if (RT_SUCCESS(rc))
2997	GCPhysCR3 = GCPhysOut;
2998	else
2999	{
3000	Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
3001	return rc;
3002	}
3003	}
3004	#endif
3005
3006	RTHCPTR HCPtrGuestCr3;
3007	int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
3008	if (RT_SUCCESS(rc))
3009	{
3010	/*
3011	* Validate the page-directory-pointer table entries (PDPE).
3012	*/
3013	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
3014	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
3015	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
3016	{
3017	/*
3018	* Map the PDPT.
3019	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
3020	* that PGMFlushTLB will be called soon and only a change to CR3 then
3021	* will cause the shadow page tables to be updated.
3022	*/
3023	#ifdef IN_RING3
3024	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
3025	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
3026	#else
3027	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
3028	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
3029	#endif
3030
3031	/*
3032	* Update CPUM and map the 4 PAE PDPEs.
3033	*/
3034	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
3035	rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
3036	if (RT_SUCCESS(rc))
3037	{
3038	#ifdef IN_RING3
3039	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
3040	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
3041	#else
3042	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
3043	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
3044	#endif
3045	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
3046	}
3047	}
3048	else
3049	rc = VERR_PGM_PAE_PDPE_RSVD;
3050	}
3051	return rc;
3052	}
3053
3054
3055	/**
3056	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
3057	*
3058	* @returns VBox status code, with the following informational code for
3059	* VM scheduling.
3060	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
3061	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
3062	*
3063	* @param pVCpu The cross context virtual CPU structure.
3064	* @param cr0 The new cr0.
3065	* @param cr4 The new cr4.
3066	* @param efer The new extended feature enable register.
3067	* @param fForce Whether to force a mode change.
3068	*/
3069	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
3070	{
3071	VMCPU_ASSERT_EMT(pVCpu);
3072
3073	/*
3074	* Calc the new guest mode.
3075	*
3076	* Note! We check PG before PE and without requiring PE because of the
3077	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
3078	*/
3079	PGMMODE enmGuestMode;
3080	if (cr0 & X86_CR0_PG)
3081	{
3082	if (!(cr4 & X86_CR4_PAE))
3083	{
3084	bool const fPse = !!(cr4 & X86_CR4_PSE);
3085	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
3086	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
3087	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
3088	enmGuestMode = PGMMODE_32_BIT;
3089	}
3090	else if (!(efer & MSR_K6_EFER_LME))
3091	{
3092	if (!(efer & MSR_K6_EFER_NXE))
3093	enmGuestMode = PGMMODE_PAE;
3094	else
3095	enmGuestMode = PGMMODE_PAE_NX;
3096	}
3097	else
3098	{
3099	if (!(efer & MSR_K6_EFER_NXE))
3100	enmGuestMode = PGMMODE_AMD64;
3101	else
3102	enmGuestMode = PGMMODE_AMD64_NX;
3103	}
3104	}
3105	else if (!(cr0 & X86_CR0_PE))
3106	enmGuestMode = PGMMODE_REAL;
3107	else
3108	enmGuestMode = PGMMODE_PROTECTED;
3109
3110	/*
3111	* Did it change?
3112	*/
3113	if ( !fForce
3114	&& pVCpu->pgm.s.enmGuestMode == enmGuestMode)
3115	return VINF_SUCCESS;
3116
3117	/* Flush the TLB */
3118	PGM_INVL_VCPU_TLBS(pVCpu);
3119	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3120	}
3121
3122
3123	/**
3124	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3125	*
3126	* @returns PGM_TYPE_*.
3127	* @param pgmMode The mode value to convert.
3128	*/
3129	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3130	{
3131	switch (pgmMode)
3132	{
3133	case PGMMODE_REAL: return PGM_TYPE_REAL;
3134	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3135	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3136	case PGMMODE_PAE:
3137	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3138	case PGMMODE_AMD64:
3139	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3140	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3141	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3142	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3143	case PGMMODE_EPT: return PGM_TYPE_EPT;
3144	case PGMMODE_NONE: return PGM_TYPE_NONE;
3145	default:
3146	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3147	}
3148	}
3149
3150
3151	/**
3152	* Calculates the shadow paging mode.
3153	*
3154	* @returns The shadow paging mode.
3155	* @param pVM The cross context VM structure.
3156	* @param enmGuestMode The guest mode.
3157	* @param enmHostMode The host mode.
3158	* @param enmShadowMode The current shadow mode.
3159	*/
3160	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3161	{
3162	switch (enmGuestMode)
3163	{
3164	case PGMMODE_REAL:
3165	case PGMMODE_PROTECTED:
3166	switch (enmHostMode)
3167	{
3168	case SUPPAGINGMODE_32_BIT:
3169	case SUPPAGINGMODE_32_BIT_GLOBAL:
3170	enmShadowMode = PGMMODE_32_BIT;
3171	break;
3172
3173	case SUPPAGINGMODE_PAE:
3174	case SUPPAGINGMODE_PAE_NX:
3175	case SUPPAGINGMODE_PAE_GLOBAL:
3176	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3177	enmShadowMode = PGMMODE_PAE;
3178	break;
3179
3180	case SUPPAGINGMODE_AMD64:
3181	case SUPPAGINGMODE_AMD64_GLOBAL:
3182	case SUPPAGINGMODE_AMD64_NX:
3183	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3184	enmShadowMode = PGMMODE_PAE;
3185	break;
3186
3187	default:
3188	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3189	}
3190	break;
3191
3192	case PGMMODE_32_BIT:
3193	switch (enmHostMode)
3194	{
3195	case SUPPAGINGMODE_32_BIT:
3196	case SUPPAGINGMODE_32_BIT_GLOBAL:
3197	enmShadowMode = PGMMODE_32_BIT;
3198	break;
3199
3200	case SUPPAGINGMODE_PAE:
3201	case SUPPAGINGMODE_PAE_NX:
3202	case SUPPAGINGMODE_PAE_GLOBAL:
3203	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3204	enmShadowMode = PGMMODE_PAE;
3205	break;
3206
3207	case SUPPAGINGMODE_AMD64:
3208	case SUPPAGINGMODE_AMD64_GLOBAL:
3209	case SUPPAGINGMODE_AMD64_NX:
3210	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3211	enmShadowMode = PGMMODE_PAE;
3212	break;
3213
3214	default:
3215	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3216	}
3217	break;
3218
3219	case PGMMODE_PAE:
3220	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3221	switch (enmHostMode)
3222	{
3223	case SUPPAGINGMODE_32_BIT:
3224	case SUPPAGINGMODE_32_BIT_GLOBAL:
3225	enmShadowMode = PGMMODE_PAE;
3226	break;
3227
3228	case SUPPAGINGMODE_PAE:
3229	case SUPPAGINGMODE_PAE_NX:
3230	case SUPPAGINGMODE_PAE_GLOBAL:
3231	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3232	enmShadowMode = PGMMODE_PAE;
3233	break;
3234
3235	case SUPPAGINGMODE_AMD64:
3236	case SUPPAGINGMODE_AMD64_GLOBAL:
3237	case SUPPAGINGMODE_AMD64_NX:
3238	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3239	enmShadowMode = PGMMODE_PAE;
3240	break;
3241
3242	default:
3243	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3244	}
3245	break;
3246
3247	case PGMMODE_AMD64:
3248	case PGMMODE_AMD64_NX:
3249	switch (enmHostMode)
3250	{
3251	case SUPPAGINGMODE_32_BIT:
3252	case SUPPAGINGMODE_32_BIT_GLOBAL:
3253	enmShadowMode = PGMMODE_AMD64;
3254	break;
3255
3256	case SUPPAGINGMODE_PAE:
3257	case SUPPAGINGMODE_PAE_NX:
3258	case SUPPAGINGMODE_PAE_GLOBAL:
3259	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3260	enmShadowMode = PGMMODE_AMD64;
3261	break;
3262
3263	case SUPPAGINGMODE_AMD64:
3264	case SUPPAGINGMODE_AMD64_GLOBAL:
3265	case SUPPAGINGMODE_AMD64_NX:
3266	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3267	enmShadowMode = PGMMODE_AMD64;
3268	break;
3269
3270	default:
3271	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3272	}
3273	break;
3274
3275	default:
3276	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3277	}
3278
3279	/*
3280	* Override the shadow mode when NEM, IEM or nested paging is active.
3281	*/
3282	if (!VM_IS_HM_ENABLED(pVM))
3283	{
3284	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
3285	pVM->pgm.s.fNestedPaging = true;
3286	enmShadowMode = PGMMODE_NONE;
3287	}
3288	else
3289	{
3290	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3291	pVM->pgm.s.fNestedPaging = fNestedPaging;
3292	if (fNestedPaging)
3293	{
3294	if (HMIsVmxActive(pVM))
3295	enmShadowMode = PGMMODE_EPT;
3296	else
3297	{
3298	/* The nested SVM paging depends on the host one. */
3299	Assert(HMIsSvmActive(pVM));
3300	if ( enmGuestMode == PGMMODE_AMD64
3301	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3302	enmShadowMode = PGMMODE_NESTED_AMD64;
3303	else
3304	switch (pVM->pgm.s.enmHostMode)
3305	{
3306	case SUPPAGINGMODE_32_BIT:
3307	case SUPPAGINGMODE_32_BIT_GLOBAL:
3308	enmShadowMode = PGMMODE_NESTED_32BIT;
3309	break;
3310
3311	case SUPPAGINGMODE_PAE:
3312	case SUPPAGINGMODE_PAE_GLOBAL:
3313	case SUPPAGINGMODE_PAE_NX:
3314	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3315	enmShadowMode = PGMMODE_NESTED_PAE;
3316	break;
3317
3318	case SUPPAGINGMODE_AMD64:
3319	case SUPPAGINGMODE_AMD64_GLOBAL:
3320	case SUPPAGINGMODE_AMD64_NX:
3321	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3322	enmShadowMode = PGMMODE_NESTED_AMD64;
3323	break;
3324
3325	default:
3326	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3327	}
3328	}
3329	}
3330	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3331	else
3332	{
3333	/* Nested paging is a requirement for nested VT-x. */
3334	AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3335	}
3336	#endif
3337	}
3338
3339	return enmShadowMode;
3340	}
3341
3342
3343	/**
3344	* Performs the actual mode change.
3345	* This is called by PGMChangeMode and pgmR3InitPaging().
3346	*
3347	* @returns VBox status code. May suspend or power off the VM on error, but this
3348	* will trigger using FFs and not informational status codes.
3349	*
3350	* @param pVM The cross context VM structure.
3351	* @param pVCpu The cross context virtual CPU structure.
3352	* @param enmGuestMode The new guest mode. This is assumed to be different from
3353	* the current mode.
3354	* @param fForce Whether to force a shadow paging mode change.
3355	*/
3356	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3357	{
3358	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3359	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3360
3361	/*
3362	* Calc the shadow mode and switcher.
3363	*/
3364	PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3365	bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode \|\| fForce;
3366
3367	/*
3368	* Exit old mode(s).
3369	*/
3370	/* shadow */
3371	if (fShadowModeChanged)
3372	{
3373	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3374	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3375	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3376	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3377	{
3378	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3379	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3380	}
3381	}
3382	else
3383	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3384
3385	/* guest */
3386	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3387	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3388	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3389	{
3390	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3391	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3392	}
3393	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3394	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3395	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3396	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3397
3398	/*
3399	* Change the paging mode data indexes.
3400	*/
3401	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3402	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3403	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3404	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3405	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3406	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3407	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3408	#ifdef IN_RING3
3409	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3410	#endif
3411
3412	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3413	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3414	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3415	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3416	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3417	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3418	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3419	#ifdef IN_RING3
3420	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3421	#endif
3422
3423	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3424	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3425	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3426	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3427	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3428	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3429	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3430	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3431	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3432	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3433	#ifdef VBOX_STRICT
3434	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3435	#endif
3436
3437	/*
3438	* Determine SLAT mode -before- entering the new shadow mode!
3439	*/
3440	pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
3441
3442	/*
3443	* Enter new shadow mode (if changed).
3444	*/
3445	if (fShadowModeChanged)
3446	{
3447	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3448	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
3449	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3450	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3451	}
3452
3453	/*
3454	* Always flag the necessary updates
3455	*/
3456	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3457
3458	/*
3459	* Enter the new guest and shadow+guest modes.
3460	*/
3461	/* Calc the new CR3 value. */
3462	RTGCPHYS GCPhysCR3;
3463	switch (enmGuestMode)
3464	{
3465	case PGMMODE_REAL:
3466	case PGMMODE_PROTECTED:
3467	GCPhysCR3 = NIL_RTGCPHYS;
3468	break;
3469
3470	case PGMMODE_32_BIT:
3471	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3472	break;
3473
3474	case PGMMODE_PAE_NX:
3475	case PGMMODE_PAE:
3476	if (!pVM->cpum.ro.GuestFeatures.fPae)
3477	#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3478	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3479	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3480	#else
3481	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3482
3483	#endif
3484	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3485	break;
3486
3487	#ifdef VBOX_WITH_64_BITS_GUESTS
3488	case PGMMODE_AMD64_NX:
3489	case PGMMODE_AMD64:
3490	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3491	break;
3492	#endif
3493	default:
3494	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3495	}
3496
3497	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3498	/*
3499	* If a nested-guest is using EPT paging:
3500	* - Update the second-level address translation (SLAT) mode.
3501	* - Indicate that the CR3 is nested-guest physical address.
3502	*/
3503	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3504	{
3505	if (PGMMODE_WITH_PAGING(enmGuestMode))
3506	{
3507	/*
3508	* Translate CR3 to its guest-physical address.
3509	* We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3510	* switching modes to keep it consistent with how GCPhysCR3 is updated.
3511	*/
3512	PGMPTWALK Walk;
3513	PGMPTWALKGST GstWalk;
3514	int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk,
3515	&GstWalk);
3516	if (RT_SUCCESS(rc))
3517	{ /* likely */ }
3518	else
3519	{
3520	/*
3521	* SLAT failed but we avoid reporting this to the caller because the caller
3522	* is not supposed to fail. The only time the caller needs to indicate a
3523	* failure to software is when PAE paging is used by the nested-guest, but
3524	* we handle the PAE case separately (e.g., see VMX transition in IEM).
3525	* In all other cases, the failure will be indicated when CR3 tries to be
3526	* translated on the next linear-address memory access.
3527	* See Intel spec. 27.2.1 "EPT Overview".
3528	*/
3529	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3530
3531	/* Trying to coax PGM to succeed for the time being... */
3532	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3533	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3534	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3535	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3536	return VINF_SUCCESS;
3537	}
3538	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3539	GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
3540	}
3541	}
3542	else
3543	Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3544	#endif
3545
3546	/*
3547	* Enter the new guest mode.
3548	*/
3549	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3550	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3551	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3552
3553	/* Set the new guest CR3 (and nested-guest CR3). */
3554	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3555
3556	/* status codes. */
3557	AssertRC(rc);
3558	AssertRC(rc2);
3559	if (RT_SUCCESS(rc))
3560	{
3561	rc = rc2;
3562	if (RT_SUCCESS(rc)) /* no informational status codes. */
3563	rc = VINF_SUCCESS;
3564	}
3565
3566	/*
3567	* Notify HM.
3568	*/
3569	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3570	return rc;
3571	}
3572
3573
3574	/**
3575	* Called by CPUM or REM when CR0.WP changes to 1.
3576	*
3577	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3578	* @thread EMT
3579	*/
3580	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3581	{
3582	/*
3583	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3584	*
3585	* Use the counter to judge whether there might be pool pages with active
3586	* hacks in them. If there are, we will be running the risk of messing up
3587	* the guest by allowing it to write to read-only pages. Thus, we have to
3588	* clear the page pool ASAP if there is the slightest chance.
3589	*/
3590	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3591	{
3592	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3593
3594	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3595	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3596	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3597	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3598	}
3599	}
3600
3601
3602	/**
3603	* Gets the current guest paging mode.
3604	*
3605	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3606	*
3607	* @returns The current paging mode.
3608	* @param pVCpu The cross context virtual CPU structure.
3609	*/
3610	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3611	{
3612	return pVCpu->pgm.s.enmGuestMode;
3613	}
3614
3615
3616	/**
3617	* Gets the current shadow paging mode.
3618	*
3619	* @returns The current paging mode.
3620	* @param pVCpu The cross context virtual CPU structure.
3621	*/
3622	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3623	{
3624	return pVCpu->pgm.s.enmShadowMode;
3625	}
3626
3627
3628	/**
3629	* Gets the current host paging mode.
3630	*
3631	* @returns The current paging mode.
3632	* @param pVM The cross context VM structure.
3633	*/
3634	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3635	{
3636	switch (pVM->pgm.s.enmHostMode)
3637	{
3638	case SUPPAGINGMODE_32_BIT:
3639	case SUPPAGINGMODE_32_BIT_GLOBAL:
3640	return PGMMODE_32_BIT;
3641
3642	case SUPPAGINGMODE_PAE:
3643	case SUPPAGINGMODE_PAE_GLOBAL:
3644	return PGMMODE_PAE;
3645
3646	case SUPPAGINGMODE_PAE_NX:
3647	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3648	return PGMMODE_PAE_NX;
3649
3650	case SUPPAGINGMODE_AMD64:
3651	case SUPPAGINGMODE_AMD64_GLOBAL:
3652	return PGMMODE_AMD64;
3653
3654	case SUPPAGINGMODE_AMD64_NX:
3655	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3656	return PGMMODE_AMD64_NX;
3657
3658	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3659	}
3660
3661	return PGMMODE_INVALID;
3662	}
3663
3664
3665	/**
3666	* Get mode name.
3667	*
3668	* @returns read-only name string.
3669	* @param enmMode The mode which name is desired.
3670	*/
3671	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3672	{
3673	switch (enmMode)
3674	{
3675	case PGMMODE_REAL: return "Real";
3676	case PGMMODE_PROTECTED: return "Protected";
3677	case PGMMODE_32_BIT: return "32-bit";
3678	case PGMMODE_PAE: return "PAE";
3679	case PGMMODE_PAE_NX: return "PAE+NX";
3680	case PGMMODE_AMD64: return "AMD64";
3681	case PGMMODE_AMD64_NX: return "AMD64+NX";
3682	case PGMMODE_NESTED_32BIT: return "Nested-32";
3683	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3684	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3685	case PGMMODE_EPT: return "EPT";
3686	case PGMMODE_NONE: return "None";
3687	default: return "unknown mode value";
3688	}
3689	}
3690
3691
3692	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3693	/**
3694	* Gets the SLAT mode name.
3695	*
3696	* @returns The read-only SLAT mode descriptive string.
3697	* @param enmSlatMode The SLAT mode value.
3698	*/
3699	VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3700	{
3701	switch (enmSlatMode)
3702	{
3703	case PGMSLAT_DIRECT: return "Direct";
3704	case PGMSLAT_EPT: return "EPT";
3705	case PGMSLAT_32BIT: return "32-bit";
3706	case PGMSLAT_PAE: return "PAE";
3707	case PGMSLAT_AMD64: return "AMD64";
3708	default: return "Unknown";
3709	}
3710	}
3711	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3712
3713
3714	/**
3715	* Gets the physical address represented in the guest CR3 as PGM sees it.
3716	*
3717	* This is mainly for logging and debugging.
3718	*
3719	* @returns PGM's guest CR3 value.
3720	* @param pVCpu The cross context virtual CPU structure.
3721	*/
3722	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3723	{
3724	return pVCpu->pgm.s.GCPhysCR3;
3725	}
3726
3727
3728
3729	/**
3730	* Notification from CPUM that the EFER.NXE bit has changed.
3731	*
3732	* @param pVCpu The cross context virtual CPU structure of the CPU for
3733	* which EFER changed.
3734	* @param fNxe The new NXE state.
3735	*/
3736	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3737	{
3738	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3739	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3740
3741	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3742	if (fNxe)
3743	{
3744	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3745	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3746	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3747	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3748	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3749	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3750	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3751	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3752	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3753	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3754	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3755
3756	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3757	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3758	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3759	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3760	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3761	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3762	}
3763	else
3764	{
3765	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3766	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3767	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3768	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3769	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3770	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3771	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3772	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3773	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3774	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3775	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3776
3777	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3778	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3779	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3780	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3781	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3782	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3783	}
3784	}
3785
3786
3787	/**
3788	* Check if any pgm pool pages are marked dirty (not monitored)
3789	*
3790	* @returns bool locked/not locked
3791	* @param pVM The cross context VM structure.
3792	*/
3793	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3794	{
3795	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3796	}
3797
3798
3799	/**
3800	* Check if this VCPU currently owns the PGM lock.
3801	*
3802	* @returns bool owner/not owner
3803	* @param pVM The cross context VM structure.
3804	*/
3805	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3806	{
3807	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3808	}
3809
3810
3811	/**
3812	* Enable or disable large page usage
3813	*
3814	* @returns VBox status code.
3815	* @param pVM The cross context VM structure.
3816	* @param fUseLargePages Use/not use large pages
3817	*/
3818	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3819	{
3820	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3821
3822	pVM->pgm.s.fUseLargePages = fUseLargePages;
3823	return VINF_SUCCESS;
3824	}
3825
3826
3827	/**
3828	* Acquire the PGM lock.
3829	*
3830	* @returns VBox status code
3831	* @param pVM The cross context VM structure.
3832	* @param fVoid Set if the caller cannot handle failure returns.
3833	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3834	*/
3835	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3836	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3837	#else
3838	int pgmLock(PVMCC pVM, bool fVoid)
3839	#endif
3840	{
3841	#if defined(VBOX_STRICT)
3842	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3843	#else
3844	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3845	#endif
3846	if (RT_SUCCESS(rc))
3847	return rc;
3848	if (fVoid)
3849	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3850	else
3851	AssertRC(rc);
3852	return rc;
3853	}
3854
3855
3856	/**
3857	* Release the PGM lock.
3858	*
3859	* @param pVM The cross context VM structure.
3860	*/
3861	void pgmUnlock(PVMCC pVM)
3862	{
3863	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3864	pVM->pgm.s.cDeprecatedPageLocks = 0;
3865	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3866	if (rc == VINF_SEM_NESTED)
3867	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3868	}
3869
3870
3871	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3872
3873	/** Format handler for PGMPAGE.
3874	* @copydoc FNRTSTRFORMATTYPE */
3875	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3876	const char pszType, void const pvValue,
3877	int cchWidth, int cchPrecision, unsigned fFlags,
3878	void *pvUser)
3879	{
3880	size_t cch;
3881	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3882	if (RT_VALID_PTR(pPage))
3883	{
3884	char szTmp[64+80];
3885
3886	cch = 0;
3887
3888	/* The single char state stuff. */
3889	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3890	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3891
3892	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3893	if (IS_PART_INCLUDED(5))
3894	{
3895	static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
3896	szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
3897	\| ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
3898	}
3899
3900	/* The type. */
3901	if (IS_PART_INCLUDED(4))
3902	{
3903	szTmp[cch++] = ':';
3904	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3905	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3906	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3907	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3908	}
3909
3910	/* The numbers. */
3911	if (IS_PART_INCLUDED(3))
3912	{
3913	szTmp[cch++] = ':';
3914	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3915	}
3916
3917	if (IS_PART_INCLUDED(2))
3918	{
3919	szTmp[cch++] = ':';
3920	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3921	}
3922
3923	if (IS_PART_INCLUDED(6))
3924	{
3925	szTmp[cch++] = ':';
3926	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3927	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3928	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3929	}
3930	# undef IS_PART_INCLUDED
3931
3932	cch = pfnOutput(pvArgOutput, szTmp, cch);
3933	}
3934	else
3935	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3936	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3937	return cch;
3938	}
3939
3940
3941	/** Format handler for PGMRAMRANGE.
3942	* @copydoc FNRTSTRFORMATTYPE */
3943	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3944	const char pszType, void const pvValue,
3945	int cchWidth, int cchPrecision, unsigned fFlags,
3946	void *pvUser)
3947	{
3948	size_t cch;
3949	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3950	if (RT_VALID_PTR(pRam))
3951	{
3952	char szTmp[80];
3953	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3954	cch = pfnOutput(pvArgOutput, szTmp, cch);
3955	}
3956	else
3957	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3958	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3959	return cch;
3960	}
3961
3962	/** Format type andlers to be registered/deregistered. */
3963	static const struct
3964	{
3965	char szType[24];
3966	PFNRTSTRFORMATTYPE pfnHandler;
3967	} g_aPgmFormatTypes[] =
3968	{
3969	{ "pgmpage", pgmFormatTypeHandlerPage },
3970	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3971	};
3972
3973	#endif /* !IN_R0 \|\| LOG_ENABLED */
3974
3975	/**
3976	* Registers the global string format types.
3977	*
3978	* This should be called at module load time or in some other manner that ensure
3979	* that it's called exactly one time.
3980	*
3981	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3982	*/
3983	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3984	{
3985	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3986	int rc = VINF_SUCCESS;
3987	unsigned i;
3988	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3989	{
3990	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3991	# ifdef IN_RING0
3992	if (rc == VERR_ALREADY_EXISTS)
3993	{
3994	/* in case of cleanup failure in ring-0 */
3995	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3996	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3997	}
3998	# endif
3999	}
4000	if (RT_FAILURE(rc))
4001	while (i-- > 0)
4002	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4003
4004	return rc;
4005	#else
4006	return VINF_SUCCESS;
4007	#endif
4008	}
4009
4010
4011	/**
4012	* Deregisters the global string format types.
4013	*
4014	* This should be called at module unload time or in some other manner that
4015	* ensure that it's called exactly one time.
4016	*/
4017	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
4018	{
4019	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4020	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4021	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4022	#endif
4023	}
4024
4025
4026	#ifdef VBOX_STRICT
4027	/**
4028	* Asserts that everything related to the guest CR3 is correctly shadowed.
4029	*
4030	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
4031	* and assert the correctness of the guest CR3 mapping before asserting that the
4032	* shadow page tables is in sync with the guest page tables.
4033	*
4034	* @returns Number of conflicts.
4035	* @param pVM The cross context VM structure.
4036	* @param pVCpu The cross context virtual CPU structure.
4037	* @param cr3 The current guest CR3 register value.
4038	* @param cr4 The current guest CR4 register value.
4039	*/
4040	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
4041	{
4042	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4043
4044	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
4045	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
4046	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
4047
4048	PGM_LOCK_VOID(pVM);
4049	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
4050	PGM_UNLOCK(pVM);
4051
4052	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4053	return cErrors;
4054	}
4055	#endif /* VBOX_STRICT */
4056
4057
4058	/**
4059	* Updates PGM's copy of the guest's EPT pointer.
4060	*
4061	* @param pVCpu The cross context virtual CPU structure.
4062	* @param uEptPtr The EPT pointer.
4063	*
4064	* @remarks This can be called as part of VM-entry so we might be in the midst of
4065	* switching to VMX non-root mode.
4066	*/
4067	VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
4068	{
4069	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4070	PGM_LOCK_VOID(pVM);
4071	pVCpu->pgm.s.uEptPtr = uEptPtr;
4072	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4073	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4074	PGM_UNLOCK(pVM);
4075	}
4076

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 100765

Download in other formats: