PGMAll.cpp@ 99654

Last change on this file since 99654 was 98103, checked in by vboxsync, 2 years ago
Copyright year updates by scm.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 147.6 KB

Line
1	/* $Id: PGMAll.cpp 98103 2023-01-17 14:15:46Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/selm.h>
37	#include <VBox/vmm/iem.h>
38	#include <VBox/vmm/iom.h>
39	#include <VBox/sup.h>
40	#include <VBox/vmm/mm.h>
41	#include <VBox/vmm/stam.h>
42	#include <VBox/vmm/trpm.h>
43	#include <VBox/vmm/em.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/vmm/hm_vmx.h>
46	#include "PGMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include "PGMInline.h"
49	#include <iprt/assert.h>
50	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
51	# include <iprt/asm-amd64-x86.h>
52	#endif
53	#include <iprt/string.h>
54	#include <VBox/log.h>
55	#include <VBox/param.h>
56	#include <VBox/err.h>
57
58
59	/*********************************************************************************************************************************
60	* Internal Functions *
61	*********************************************************************************************************************************/
62	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
63	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
64	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
65	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
66	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
67	PPGMPTWALKGST pGstWalk);
68	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
69	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
70	PPGMPTWALKGST pGstWalkAll);
71	#endif
72	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
73	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
74
75
76	/*
77	* Second level transation - EPT.
78	*/
79	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
80	# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
81	# include "PGMSlatDefs.h"
82	# include "PGMAllGstSlatEpt.cpp.h"
83	# undef PGM_SLAT_TYPE
84	#endif
85
86
87	/*
88	* Shadow - 32-bit mode
89	*/
90	#define PGM_SHW_TYPE PGM_TYPE_32BIT
91	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
92	#include "PGMAllShw.h"
93
94	/* Guest - real mode */
95	#define PGM_GST_TYPE PGM_TYPE_REAL
96	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
97	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
98	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
99	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
100	#include "PGMGstDefs.h"
101	#include "PGMAllGst.h"
102	#include "PGMAllBth.h"
103	#undef BTH_PGMPOOLKIND_PT_FOR_PT
104	#undef BTH_PGMPOOLKIND_ROOT
105	#undef PGM_BTH_NAME
106	#undef PGM_GST_TYPE
107	#undef PGM_GST_NAME
108
109	/* Guest - protected mode */
110	#define PGM_GST_TYPE PGM_TYPE_PROT
111	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
112	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
113	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
114	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
115	#include "PGMGstDefs.h"
116	#include "PGMAllGst.h"
117	#include "PGMAllBth.h"
118	#undef BTH_PGMPOOLKIND_PT_FOR_PT
119	#undef BTH_PGMPOOLKIND_ROOT
120	#undef PGM_BTH_NAME
121	#undef PGM_GST_TYPE
122	#undef PGM_GST_NAME
123
124	/* Guest - 32-bit mode */
125	#define PGM_GST_TYPE PGM_TYPE_32BIT
126	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
127	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
128	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
129	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
130	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
131	#include "PGMGstDefs.h"
132	#include "PGMAllGst.h"
133	#include "PGMAllBth.h"
134	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
135	#undef BTH_PGMPOOLKIND_PT_FOR_PT
136	#undef BTH_PGMPOOLKIND_ROOT
137	#undef PGM_BTH_NAME
138	#undef PGM_GST_TYPE
139	#undef PGM_GST_NAME
140
141	#undef PGM_SHW_TYPE
142	#undef PGM_SHW_NAME
143
144
145	/*
146	* Shadow - PAE mode
147	*/
148	#define PGM_SHW_TYPE PGM_TYPE_PAE
149	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
150	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
151	#include "PGMAllShw.h"
152
153	/* Guest - real mode */
154	#define PGM_GST_TYPE PGM_TYPE_REAL
155	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
156	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
157	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
158	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
159	#include "PGMGstDefs.h"
160	#include "PGMAllBth.h"
161	#undef BTH_PGMPOOLKIND_PT_FOR_PT
162	#undef BTH_PGMPOOLKIND_ROOT
163	#undef PGM_BTH_NAME
164	#undef PGM_GST_TYPE
165	#undef PGM_GST_NAME
166
167	/* Guest - protected mode */
168	#define PGM_GST_TYPE PGM_TYPE_PROT
169	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
170	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
171	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
172	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
173	#include "PGMGstDefs.h"
174	#include "PGMAllBth.h"
175	#undef BTH_PGMPOOLKIND_PT_FOR_PT
176	#undef BTH_PGMPOOLKIND_ROOT
177	#undef PGM_BTH_NAME
178	#undef PGM_GST_TYPE
179	#undef PGM_GST_NAME
180
181	/* Guest - 32-bit mode */
182	#define PGM_GST_TYPE PGM_TYPE_32BIT
183	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
184	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
185	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
186	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
187	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
188	#include "PGMGstDefs.h"
189	#include "PGMAllBth.h"
190	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
191	#undef BTH_PGMPOOLKIND_PT_FOR_PT
192	#undef BTH_PGMPOOLKIND_ROOT
193	#undef PGM_BTH_NAME
194	#undef PGM_GST_TYPE
195	#undef PGM_GST_NAME
196
197
198	/* Guest - PAE mode */
199	#define PGM_GST_TYPE PGM_TYPE_PAE
200	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
201	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
202	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
203	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
204	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
205	#include "PGMGstDefs.h"
206	#include "PGMAllGst.h"
207	#include "PGMAllBth.h"
208	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
209	#undef BTH_PGMPOOLKIND_PT_FOR_PT
210	#undef BTH_PGMPOOLKIND_ROOT
211	#undef PGM_BTH_NAME
212	#undef PGM_GST_TYPE
213	#undef PGM_GST_NAME
214
215	#undef PGM_SHW_TYPE
216	#undef PGM_SHW_NAME
217
218
219	/*
220	* Shadow - AMD64 mode
221	*/
222	#define PGM_SHW_TYPE PGM_TYPE_AMD64
223	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
224	#include "PGMAllShw.h"
225
226	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
227	/** @todo retire this hack. */
228	#define PGM_GST_TYPE PGM_TYPE_PROT
229	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
230	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
231	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
232	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
233	#include "PGMGstDefs.h"
234	#include "PGMAllBth.h"
235	#undef BTH_PGMPOOLKIND_PT_FOR_PT
236	#undef BTH_PGMPOOLKIND_ROOT
237	#undef PGM_BTH_NAME
238	#undef PGM_GST_TYPE
239	#undef PGM_GST_NAME
240
241	#ifdef VBOX_WITH_64_BITS_GUESTS
242	/* Guest - AMD64 mode */
243	# define PGM_GST_TYPE PGM_TYPE_AMD64
244	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
245	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
246	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
247	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
248	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
249	# include "PGMGstDefs.h"
250	# include "PGMAllGst.h"
251	# include "PGMAllBth.h"
252	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
253	# undef BTH_PGMPOOLKIND_PT_FOR_PT
254	# undef BTH_PGMPOOLKIND_ROOT
255	# undef PGM_BTH_NAME
256	# undef PGM_GST_TYPE
257	# undef PGM_GST_NAME
258	#endif /* VBOX_WITH_64_BITS_GUESTS */
259
260	#undef PGM_SHW_TYPE
261	#undef PGM_SHW_NAME
262
263
264	/*
265	* Shadow - 32-bit nested paging mode.
266	*/
267	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
268	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
269	#include "PGMAllShw.h"
270
271	/* Guest - real mode */
272	#define PGM_GST_TYPE PGM_TYPE_REAL
273	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
274	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
275	#include "PGMGstDefs.h"
276	#include "PGMAllBth.h"
277	#undef PGM_BTH_NAME
278	#undef PGM_GST_TYPE
279	#undef PGM_GST_NAME
280
281	/* Guest - protected mode */
282	#define PGM_GST_TYPE PGM_TYPE_PROT
283	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
284	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
285	#include "PGMGstDefs.h"
286	#include "PGMAllBth.h"
287	#undef PGM_BTH_NAME
288	#undef PGM_GST_TYPE
289	#undef PGM_GST_NAME
290
291	/* Guest - 32-bit mode */
292	#define PGM_GST_TYPE PGM_TYPE_32BIT
293	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
294	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
295	#include "PGMGstDefs.h"
296	#include "PGMAllBth.h"
297	#undef PGM_BTH_NAME
298	#undef PGM_GST_TYPE
299	#undef PGM_GST_NAME
300
301	/* Guest - PAE mode */
302	#define PGM_GST_TYPE PGM_TYPE_PAE
303	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
304	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
305	#include "PGMGstDefs.h"
306	#include "PGMAllBth.h"
307	#undef PGM_BTH_NAME
308	#undef PGM_GST_TYPE
309	#undef PGM_GST_NAME
310
311	#ifdef VBOX_WITH_64_BITS_GUESTS
312	/* Guest - AMD64 mode */
313	# define PGM_GST_TYPE PGM_TYPE_AMD64
314	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
315	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
316	# include "PGMGstDefs.h"
317	# include "PGMAllBth.h"
318	# undef PGM_BTH_NAME
319	# undef PGM_GST_TYPE
320	# undef PGM_GST_NAME
321	#endif /* VBOX_WITH_64_BITS_GUESTS */
322
323	#undef PGM_SHW_TYPE
324	#undef PGM_SHW_NAME
325
326
327	/*
328	* Shadow - PAE nested paging mode.
329	*/
330	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
331	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
332	#include "PGMAllShw.h"
333
334	/* Guest - real mode */
335	#define PGM_GST_TYPE PGM_TYPE_REAL
336	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
337	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
338	#include "PGMGstDefs.h"
339	#include "PGMAllBth.h"
340	#undef PGM_BTH_NAME
341	#undef PGM_GST_TYPE
342	#undef PGM_GST_NAME
343
344	/* Guest - protected mode */
345	#define PGM_GST_TYPE PGM_TYPE_PROT
346	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
347	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
348	#include "PGMGstDefs.h"
349	#include "PGMAllBth.h"
350	#undef PGM_BTH_NAME
351	#undef PGM_GST_TYPE
352	#undef PGM_GST_NAME
353
354	/* Guest - 32-bit mode */
355	#define PGM_GST_TYPE PGM_TYPE_32BIT
356	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
357	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
358	#include "PGMGstDefs.h"
359	#include "PGMAllBth.h"
360	#undef PGM_BTH_NAME
361	#undef PGM_GST_TYPE
362	#undef PGM_GST_NAME
363
364	/* Guest - PAE mode */
365	#define PGM_GST_TYPE PGM_TYPE_PAE
366	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
367	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
368	#include "PGMGstDefs.h"
369	#include "PGMAllBth.h"
370	#undef PGM_BTH_NAME
371	#undef PGM_GST_TYPE
372	#undef PGM_GST_NAME
373
374	#ifdef VBOX_WITH_64_BITS_GUESTS
375	/* Guest - AMD64 mode */
376	# define PGM_GST_TYPE PGM_TYPE_AMD64
377	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
378	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
379	# include "PGMGstDefs.h"
380	# include "PGMAllBth.h"
381	# undef PGM_BTH_NAME
382	# undef PGM_GST_TYPE
383	# undef PGM_GST_NAME
384	#endif /* VBOX_WITH_64_BITS_GUESTS */
385
386	#undef PGM_SHW_TYPE
387	#undef PGM_SHW_NAME
388
389
390	/*
391	* Shadow - AMD64 nested paging mode.
392	*/
393	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
394	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
395	#include "PGMAllShw.h"
396
397	/* Guest - real mode */
398	#define PGM_GST_TYPE PGM_TYPE_REAL
399	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
400	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
401	#include "PGMGstDefs.h"
402	#include "PGMAllBth.h"
403	#undef PGM_BTH_NAME
404	#undef PGM_GST_TYPE
405	#undef PGM_GST_NAME
406
407	/* Guest - protected mode */
408	#define PGM_GST_TYPE PGM_TYPE_PROT
409	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
410	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
411	#include "PGMGstDefs.h"
412	#include "PGMAllBth.h"
413	#undef PGM_BTH_NAME
414	#undef PGM_GST_TYPE
415	#undef PGM_GST_NAME
416
417	/* Guest - 32-bit mode */
418	#define PGM_GST_TYPE PGM_TYPE_32BIT
419	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
420	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
421	#include "PGMGstDefs.h"
422	#include "PGMAllBth.h"
423	#undef PGM_BTH_NAME
424	#undef PGM_GST_TYPE
425	#undef PGM_GST_NAME
426
427	/* Guest - PAE mode */
428	#define PGM_GST_TYPE PGM_TYPE_PAE
429	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
430	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
431	#include "PGMGstDefs.h"
432	#include "PGMAllBth.h"
433	#undef PGM_BTH_NAME
434	#undef PGM_GST_TYPE
435	#undef PGM_GST_NAME
436
437	#ifdef VBOX_WITH_64_BITS_GUESTS
438	/* Guest - AMD64 mode */
439	# define PGM_GST_TYPE PGM_TYPE_AMD64
440	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
441	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
442	# include "PGMGstDefs.h"
443	# include "PGMAllBth.h"
444	# undef PGM_BTH_NAME
445	# undef PGM_GST_TYPE
446	# undef PGM_GST_NAME
447	#endif /* VBOX_WITH_64_BITS_GUESTS */
448
449	#undef PGM_SHW_TYPE
450	#undef PGM_SHW_NAME
451
452
453	/*
454	* Shadow - EPT.
455	*/
456	#define PGM_SHW_TYPE PGM_TYPE_EPT
457	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
458	#include "PGMAllShw.h"
459
460	/* Guest - real mode */
461	#define PGM_GST_TYPE PGM_TYPE_REAL
462	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
463	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
464	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
465	#include "PGMGstDefs.h"
466	#include "PGMAllBth.h"
467	#undef BTH_PGMPOOLKIND_PT_FOR_PT
468	#undef PGM_BTH_NAME
469	#undef PGM_GST_TYPE
470	#undef PGM_GST_NAME
471
472	/* Guest - protected mode */
473	#define PGM_GST_TYPE PGM_TYPE_PROT
474	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
475	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
476	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
477	#include "PGMGstDefs.h"
478	#include "PGMAllBth.h"
479	#undef BTH_PGMPOOLKIND_PT_FOR_PT
480	#undef PGM_BTH_NAME
481	#undef PGM_GST_TYPE
482	#undef PGM_GST_NAME
483
484	/* Guest - 32-bit mode */
485	#define PGM_GST_TYPE PGM_TYPE_32BIT
486	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
487	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
488	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
489	#include "PGMGstDefs.h"
490	#include "PGMAllBth.h"
491	#undef BTH_PGMPOOLKIND_PT_FOR_PT
492	#undef PGM_BTH_NAME
493	#undef PGM_GST_TYPE
494	#undef PGM_GST_NAME
495
496	/* Guest - PAE mode */
497	#define PGM_GST_TYPE PGM_TYPE_PAE
498	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
499	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
500	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
501	#include "PGMGstDefs.h"
502	#include "PGMAllBth.h"
503	#undef BTH_PGMPOOLKIND_PT_FOR_PT
504	#undef PGM_BTH_NAME
505	#undef PGM_GST_TYPE
506	#undef PGM_GST_NAME
507
508	#ifdef VBOX_WITH_64_BITS_GUESTS
509	/* Guest - AMD64 mode */
510	# define PGM_GST_TYPE PGM_TYPE_AMD64
511	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
512	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
513	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
514	# include "PGMGstDefs.h"
515	# include "PGMAllBth.h"
516	# undef BTH_PGMPOOLKIND_PT_FOR_PT
517	# undef PGM_BTH_NAME
518	# undef PGM_GST_TYPE
519	# undef PGM_GST_NAME
520	#endif /* VBOX_WITH_64_BITS_GUESTS */
521
522	#undef PGM_SHW_TYPE
523	#undef PGM_SHW_NAME
524
525
526	/*
527	* Shadow - NEM / None.
528	*/
529	#define PGM_SHW_TYPE PGM_TYPE_NONE
530	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
531	#include "PGMAllShw.h"
532
533	/* Guest - real mode */
534	#define PGM_GST_TYPE PGM_TYPE_REAL
535	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
536	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
537	#include "PGMGstDefs.h"
538	#include "PGMAllBth.h"
539	#undef PGM_BTH_NAME
540	#undef PGM_GST_TYPE
541	#undef PGM_GST_NAME
542
543	/* Guest - protected mode */
544	#define PGM_GST_TYPE PGM_TYPE_PROT
545	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
546	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
547	#include "PGMGstDefs.h"
548	#include "PGMAllBth.h"
549	#undef PGM_BTH_NAME
550	#undef PGM_GST_TYPE
551	#undef PGM_GST_NAME
552
553	/* Guest - 32-bit mode */
554	#define PGM_GST_TYPE PGM_TYPE_32BIT
555	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
556	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
557	#include "PGMGstDefs.h"
558	#include "PGMAllBth.h"
559	#undef PGM_BTH_NAME
560	#undef PGM_GST_TYPE
561	#undef PGM_GST_NAME
562
563	/* Guest - PAE mode */
564	#define PGM_GST_TYPE PGM_TYPE_PAE
565	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
566	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
567	#include "PGMGstDefs.h"
568	#include "PGMAllBth.h"
569	#undef PGM_BTH_NAME
570	#undef PGM_GST_TYPE
571	#undef PGM_GST_NAME
572
573	#ifdef VBOX_WITH_64_BITS_GUESTS
574	/* Guest - AMD64 mode */
575	# define PGM_GST_TYPE PGM_TYPE_AMD64
576	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
577	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
578	# include "PGMGstDefs.h"
579	# include "PGMAllBth.h"
580	# undef PGM_BTH_NAME
581	# undef PGM_GST_TYPE
582	# undef PGM_GST_NAME
583	#endif /* VBOX_WITH_64_BITS_GUESTS */
584
585	#undef PGM_SHW_TYPE
586	#undef PGM_SHW_NAME
587
588
589
590	/**
591	* Guest mode data array.
592	*/
593	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
594	{
595	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
596	{
597	PGM_TYPE_REAL,
598	PGM_GST_NAME_REAL(GetPage),
599	PGM_GST_NAME_REAL(ModifyPage),
600	PGM_GST_NAME_REAL(Enter),
601	PGM_GST_NAME_REAL(Exit),
602	#ifdef IN_RING3
603	PGM_GST_NAME_REAL(Relocate),
604	#endif
605	},
606	{
607	PGM_TYPE_PROT,
608	PGM_GST_NAME_PROT(GetPage),
609	PGM_GST_NAME_PROT(ModifyPage),
610	PGM_GST_NAME_PROT(Enter),
611	PGM_GST_NAME_PROT(Exit),
612	#ifdef IN_RING3
613	PGM_GST_NAME_PROT(Relocate),
614	#endif
615	},
616	{
617	PGM_TYPE_32BIT,
618	PGM_GST_NAME_32BIT(GetPage),
619	PGM_GST_NAME_32BIT(ModifyPage),
620	PGM_GST_NAME_32BIT(Enter),
621	PGM_GST_NAME_32BIT(Exit),
622	#ifdef IN_RING3
623	PGM_GST_NAME_32BIT(Relocate),
624	#endif
625	},
626	{
627	PGM_TYPE_PAE,
628	PGM_GST_NAME_PAE(GetPage),
629	PGM_GST_NAME_PAE(ModifyPage),
630	PGM_GST_NAME_PAE(Enter),
631	PGM_GST_NAME_PAE(Exit),
632	#ifdef IN_RING3
633	PGM_GST_NAME_PAE(Relocate),
634	#endif
635	},
636	#ifdef VBOX_WITH_64_BITS_GUESTS
637	{
638	PGM_TYPE_AMD64,
639	PGM_GST_NAME_AMD64(GetPage),
640	PGM_GST_NAME_AMD64(ModifyPage),
641	PGM_GST_NAME_AMD64(Enter),
642	PGM_GST_NAME_AMD64(Exit),
643	# ifdef IN_RING3
644	PGM_GST_NAME_AMD64(Relocate),
645	# endif
646	},
647	#endif
648	};
649
650
651	/**
652	* The shadow mode data array.
653	*/
654	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
655	{
656	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
657	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
658	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
659	{
660	PGM_TYPE_32BIT,
661	PGM_SHW_NAME_32BIT(GetPage),
662	PGM_SHW_NAME_32BIT(ModifyPage),
663	PGM_SHW_NAME_32BIT(Enter),
664	PGM_SHW_NAME_32BIT(Exit),
665	#ifdef IN_RING3
666	PGM_SHW_NAME_32BIT(Relocate),
667	#endif
668	},
669	{
670	PGM_TYPE_PAE,
671	PGM_SHW_NAME_PAE(GetPage),
672	PGM_SHW_NAME_PAE(ModifyPage),
673	PGM_SHW_NAME_PAE(Enter),
674	PGM_SHW_NAME_PAE(Exit),
675	#ifdef IN_RING3
676	PGM_SHW_NAME_PAE(Relocate),
677	#endif
678	},
679	{
680	PGM_TYPE_AMD64,
681	PGM_SHW_NAME_AMD64(GetPage),
682	PGM_SHW_NAME_AMD64(ModifyPage),
683	PGM_SHW_NAME_AMD64(Enter),
684	PGM_SHW_NAME_AMD64(Exit),
685	#ifdef IN_RING3
686	PGM_SHW_NAME_AMD64(Relocate),
687	#endif
688	},
689	{
690	PGM_TYPE_NESTED_32BIT,
691	PGM_SHW_NAME_NESTED_32BIT(GetPage),
692	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
693	PGM_SHW_NAME_NESTED_32BIT(Enter),
694	PGM_SHW_NAME_NESTED_32BIT(Exit),
695	#ifdef IN_RING3
696	PGM_SHW_NAME_NESTED_32BIT(Relocate),
697	#endif
698	},
699	{
700	PGM_TYPE_NESTED_PAE,
701	PGM_SHW_NAME_NESTED_PAE(GetPage),
702	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
703	PGM_SHW_NAME_NESTED_PAE(Enter),
704	PGM_SHW_NAME_NESTED_PAE(Exit),
705	#ifdef IN_RING3
706	PGM_SHW_NAME_NESTED_PAE(Relocate),
707	#endif
708	},
709	{
710	PGM_TYPE_NESTED_AMD64,
711	PGM_SHW_NAME_NESTED_AMD64(GetPage),
712	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
713	PGM_SHW_NAME_NESTED_AMD64(Enter),
714	PGM_SHW_NAME_NESTED_AMD64(Exit),
715	#ifdef IN_RING3
716	PGM_SHW_NAME_NESTED_AMD64(Relocate),
717	#endif
718	},
719	{
720	PGM_TYPE_EPT,
721	PGM_SHW_NAME_EPT(GetPage),
722	PGM_SHW_NAME_EPT(ModifyPage),
723	PGM_SHW_NAME_EPT(Enter),
724	PGM_SHW_NAME_EPT(Exit),
725	#ifdef IN_RING3
726	PGM_SHW_NAME_EPT(Relocate),
727	#endif
728	},
729	{
730	PGM_TYPE_NONE,
731	PGM_SHW_NAME_NONE(GetPage),
732	PGM_SHW_NAME_NONE(ModifyPage),
733	PGM_SHW_NAME_NONE(Enter),
734	PGM_SHW_NAME_NONE(Exit),
735	#ifdef IN_RING3
736	PGM_SHW_NAME_NONE(Relocate),
737	#endif
738	},
739	};
740
741
742	/**
743	* The guest+shadow mode data array.
744	*/
745	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
746	{
747	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
748	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
749	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
750	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
751
752	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
753	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
754	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
755	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
756
757	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
758	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
759	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
760	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
761
762	#elif defined(IN_RING3) && defined(VBOX_STRICT)
763	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
764	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
765	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
766
767	#else
768	# error "Misconfig."
769	#endif
770
771	/* 32-bit shadow paging mode: */
772	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
773	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
774	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
775	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
776	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
777	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
778	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
780	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
781	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
782	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
783
784	/* PAE shadow paging mode: */
785	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
786	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
787	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
788	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
789	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
790	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
791	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
793	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
796
797	/* AMD64 shadow paging mode: */
798	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
799	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
800	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
801	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
802	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
803	#ifdef VBOX_WITH_64_BITS_GUESTS
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
805	#else
806	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
807	#endif
808	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
810	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
811	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
812	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
813
814	/* 32-bit nested paging mode: */
815	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
816	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
817	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
818	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
820	#ifdef VBOX_WITH_64_BITS_GUESTS
821	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
822	#else
823	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
824	#endif
825	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
827	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
828	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
829	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
830
831	/* PAE nested paging mode: */
832	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
833	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
834	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
835	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
837	#ifdef VBOX_WITH_64_BITS_GUESTS
838	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
839	#else
840	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
841	#endif
842	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
844	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
845	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
846	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
847
848	/* AMD64 nested paging mode: */
849	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
850	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
851	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
852	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
854	#ifdef VBOX_WITH_64_BITS_GUESTS
855	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
856	#else
857	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
858	#endif
859	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
861	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
862	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
863	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
864
865	/* EPT nested paging mode: */
866	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
867	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
868	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
869	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
870	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
871	#ifdef VBOX_WITH_64_BITS_GUESTS
872	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
873	#else
874	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
875	#endif
876	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
877	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
878	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
879	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
880	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
881
882	/* NONE / NEM: */
883	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
884	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
885	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
886	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
887	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
888	#ifdef VBOX_WITH_64_BITS_GUESTS
889	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
890	#else
891	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
892	#endif
893	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
894	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
895	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
896	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
897	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
898
899
900	#undef PGMMODEDATABTH_ENTRY
901	#undef PGMMODEDATABTH_NULL_ENTRY
902	};
903
904
905	/** Mask array used by pgmGetCr3MaskForMode.
906	* X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
907	static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
908	{
909	/* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK,
910	/* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK,
911	/* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK,
912	/* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK,
913	/* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK,
914	/* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK,
915	/* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
916	/* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK,
917	/* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK,
918	/* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK,
919	/* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
920	/* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK,
921	/* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK,
922	};
923
924
925	/**
926	* Gets the physical address mask for CR3 in the given paging mode.
927	*
928	* The mask is for eliminating flags and other stuff in CR3/EPTP when
929	* extracting the physical address. It is not for validating whether there are
930	* reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed
931	* it to PGM checked for reserved bits, including reserved physical address
932	* bits.
933	*
934	* @returns The CR3 mask.
935	* @param enmMode The paging mode.
936	* @param enmSlatMode The second-level address translation mode.
937	*/
938	DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
939	{
940	if (enmSlatMode == PGMSLAT_DIRECT)
941	{
942	Assert(enmMode != PGMMODE_EPT);
943	return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
944	}
945	Assert(enmSlatMode == PGMSLAT_EPT);
946	return X86_CR3_EPT_PAGE_MASK;
947	}
948
949
950	/**
951	* Gets the masked CR3 value according to the current guest paging mode.
952	*
953	* See disclaimer in pgmGetCr3MaskForMode.
954	*
955	* @returns The masked PGM CR3 value.
956	* @param pVCpu The cross context virtual CPU structure.
957	* @param uCr3 The raw guest CR3 value.
958	*/
959	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
960	{
961	uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
962	RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
963	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
964	return GCPhysCR3;
965	}
966
967
968	#ifdef IN_RING0
969	/**
970	* #PF Handler.
971	*
972	* @returns VBox status code (appropriate for trap handling and GC return).
973	* @param pVCpu The cross context virtual CPU structure.
974	* @param uErr The trap error code.
975	* @param pCtx Pointer to the register context for the CPU.
976	* @param pvFault The fault address.
977	*/
978	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
979	{
980	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
981
982	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
983	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
984	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
985
986
987	# ifdef VBOX_WITH_STATISTICS
988	/*
989	* Error code stats.
990	*/
991	if (uErr & X86_TRAP_PF_US)
992	{
993	if (!(uErr & X86_TRAP_PF_P))
994	{
995	if (uErr & X86_TRAP_PF_RW)
996	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
997	else
998	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
999	}
1000	else if (uErr & X86_TRAP_PF_RW)
1001	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1002	else if (uErr & X86_TRAP_PF_RSVD)
1003	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1004	else if (uErr & X86_TRAP_PF_ID)
1005	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1006	else
1007	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1008	}
1009	else
1010	{ /* Supervisor */
1011	if (!(uErr & X86_TRAP_PF_P))
1012	{
1013	if (uErr & X86_TRAP_PF_RW)
1014	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1015	else
1016	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1017	}
1018	else if (uErr & X86_TRAP_PF_RW)
1019	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1020	else if (uErr & X86_TRAP_PF_ID)
1021	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1022	else if (uErr & X86_TRAP_PF_RSVD)
1023	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1024	}
1025	# endif /* VBOX_WITH_STATISTICS */
1026
1027	/*
1028	* Call the worker.
1029	*/
1030	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1031	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1032	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
1033	bool fLockTaken = false;
1034	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
1035	if (fLockTaken)
1036	{
1037	PGM_LOCK_ASSERT_OWNER(pVM);
1038	PGM_UNLOCK(pVM);
1039	}
1040	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
1041
1042	/*
1043	* Return code tweaks.
1044	*/
1045	if (rc != VINF_SUCCESS)
1046	{
1047	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1048	rc = VINF_SUCCESS;
1049
1050	/* Note: hack alert for difficult to reproduce problem. */
1051	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1052	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1053	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1054	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1055	{
1056	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
1057	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
1058	rc = VINF_SUCCESS;
1059	}
1060	}
1061
1062	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
1063	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1064	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1065	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1066	return rc;
1067	}
1068	#endif /* IN_RING0 */
1069
1070
1071	/**
1072	* Prefetch a page
1073	*
1074	* Typically used to sync commonly used pages before entering raw mode
1075	* after a CR3 reload.
1076	*
1077	* @returns VBox status code suitable for scheduling.
1078	* @retval VINF_SUCCESS on success.
1079	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1080	* @param pVCpu The cross context virtual CPU structure.
1081	* @param GCPtrPage Page to invalidate.
1082	*/
1083	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1084	{
1085	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1086
1087	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1088	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1089	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1090	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1091
1092	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1093	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1094	return rc;
1095	}
1096
1097
1098	/**
1099	* Emulation of the invlpg instruction (HC only actually).
1100	*
1101	* @returns Strict VBox status code, special care required.
1102	* @retval VINF_PGM_SYNC_CR3 - handled.
1103	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1104	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1105	*
1106	* @param pVCpu The cross context virtual CPU structure.
1107	* @param GCPtrPage Page to invalidate.
1108	*
1109	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1110	* safe, but there could be edge cases!
1111	*
1112	* @todo Flush page or page directory only if necessary!
1113	* @todo VBOXSTRICTRC
1114	*/
1115	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1116	{
1117	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1118	int rc;
1119	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1120
1121	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1122
1123	/*
1124	* Call paging mode specific worker.
1125	*/
1126	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1127	PGM_LOCK_VOID(pVM);
1128
1129	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1130	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1131	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1132	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1133
1134	PGM_UNLOCK(pVM);
1135	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1136
1137	/* Ignore all irrelevant error codes. */
1138	if ( rc == VERR_PAGE_NOT_PRESENT
1139	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1140	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1141	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1142	rc = VINF_SUCCESS;
1143
1144	return rc;
1145	}
1146
1147
1148	/**
1149	* Executes an instruction using the interpreter.
1150	*
1151	* @returns VBox status code (appropriate for trap handling and GC return).
1152	* @param pVCpu The cross context virtual CPU structure.
1153	* @param pvFault Fault address.
1154	*/
1155	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
1156	{
1157	RT_NOREF(pvFault);
1158	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
1159	if (rc == VERR_EM_INTERPRETER)
1160	rc = VINF_EM_RAW_EMULATE_INSTR;
1161	if (rc != VINF_SUCCESS)
1162	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1163	return rc;
1164	}
1165
1166
1167	/**
1168	* Gets effective page information (from the VMM page directory).
1169	*
1170	* @returns VBox status code.
1171	* @param pVCpu The cross context virtual CPU structure.
1172	* @param GCPtr Guest Context virtual address of the page.
1173	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1174	* @param pHCPhys Where to store the HC physical address of the page.
1175	* This is page aligned.
1176	* @remark You should use PGMMapGetPage() for pages in a mapping.
1177	*/
1178	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1179	{
1180	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1181	PGM_LOCK_VOID(pVM);
1182
1183	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1184	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1185	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1186	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1187
1188	PGM_UNLOCK(pVM);
1189	return rc;
1190	}
1191
1192
1193	/**
1194	* Modify page flags for a range of pages in the shadow context.
1195	*
1196	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1197	*
1198	* @returns VBox status code.
1199	* @param pVCpu The cross context virtual CPU structure.
1200	* @param GCPtr Virtual address of the first page in the range.
1201	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1202	* @param fMask The AND mask - page flags X86_PTE_*.
1203	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1204	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1205	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1206	*/
1207	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1208	{
1209	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1210	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1211
1212	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1213
1214	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1215	PGM_LOCK_VOID(pVM);
1216
1217	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1218	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1219	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1220	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1221
1222	PGM_UNLOCK(pVM);
1223	return rc;
1224	}
1225
1226
1227	/**
1228	* Changing the page flags for a single page in the shadow page tables so as to
1229	* make it read-only.
1230	*
1231	* @returns VBox status code.
1232	* @param pVCpu The cross context virtual CPU structure.
1233	* @param GCPtr Virtual address of the first page in the range.
1234	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1235	*/
1236	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1237	{
1238	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1239	}
1240
1241
1242	/**
1243	* Changing the page flags for a single page in the shadow page tables so as to
1244	* make it writable.
1245	*
1246	* The call must know with 101% certainty that the guest page tables maps this
1247	* as writable too. This function will deal shared, zero and write monitored
1248	* pages.
1249	*
1250	* @returns VBox status code.
1251	* @param pVCpu The cross context virtual CPU structure.
1252	* @param GCPtr Virtual address of the first page in the range.
1253	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1254	*/
1255	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1256	{
1257	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1258	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1259	return VINF_SUCCESS;
1260	}
1261
1262
1263	/**
1264	* Changing the page flags for a single page in the shadow page tables so as to
1265	* make it not present.
1266	*
1267	* @returns VBox status code.
1268	* @param pVCpu The cross context virtual CPU structure.
1269	* @param GCPtr Virtual address of the first page in the range.
1270	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1271	*/
1272	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1273	{
1274	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1275	}
1276
1277
1278	/**
1279	* Changing the page flags for a single page in the shadow page tables so as to
1280	* make it supervisor and writable.
1281	*
1282	* This if for dealing with CR0.WP=0 and readonly user pages.
1283	*
1284	* @returns VBox status code.
1285	* @param pVCpu The cross context virtual CPU structure.
1286	* @param GCPtr Virtual address of the first page in the range.
1287	* @param fBigPage Whether or not this is a big page. If it is, we have to
1288	* change the shadow PDE as well. If it isn't, the caller
1289	* has checked that the shadow PDE doesn't need changing.
1290	* We ASSUME 4KB pages backing the big page here!
1291	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1292	*/
1293	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1294	{
1295	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1296	if (rc == VINF_SUCCESS && fBigPage)
1297	{
1298	/* this is a bit ugly... */
1299	switch (pVCpu->pgm.s.enmShadowMode)
1300	{
1301	case PGMMODE_32_BIT:
1302	{
1303	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1304	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1305	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1306	pPde->u \|= X86_PDE_RW;
1307	Log(("-> PDE=%#llx (32)\n", pPde->u));
1308	break;
1309	}
1310	case PGMMODE_PAE:
1311	case PGMMODE_PAE_NX:
1312	{
1313	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1314	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1315	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1316	pPde->u \|= X86_PDE_RW;
1317	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1318	break;
1319	}
1320	default:
1321	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1322	}
1323	}
1324	return rc;
1325	}
1326
1327
1328	/**
1329	* Gets the shadow page directory for the specified address, PAE.
1330	*
1331	* @returns Pointer to the shadow PD.
1332	* @param pVCpu The cross context virtual CPU structure.
1333	* @param GCPtr The address.
1334	* @param uGstPdpe Guest PDPT entry. Valid.
1335	* @param ppPD Receives address of page directory
1336	*/
1337	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1338	{
1339	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1340	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1341	PPGMPOOLPAGE pShwPage;
1342	int rc;
1343	PGM_LOCK_ASSERT_OWNER(pVM);
1344
1345
1346	/* Allocate page directory if not present. */
1347	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1348	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1349	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1350	X86PGPAEUINT const uPdpe = pPdpe->u;
1351	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1352	{
1353	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1354	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1355	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1356
1357	pgmPoolCacheUsed(pPool, pShwPage);
1358
1359	/* Update the entry if necessary. */
1360	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1361	if (uPdpeNew == uPdpe)
1362	{ /* likely */ }
1363	else
1364	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1365	}
1366	else
1367	{
1368	RTGCPTR64 GCPdPt;
1369	PGMPOOLKIND enmKind;
1370	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1371	{
1372	/* AMD-V nested paging or real/protected mode without paging. */
1373	GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1374	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1375	}
1376	else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1377	{
1378	if (uGstPdpe & X86_PDPE_P)
1379	{
1380	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1381	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1382	}
1383	else
1384	{
1385	/* PD not present; guest must reload CR3 to change it.
1386	* No need to monitor anything in this case. */
1387	/** @todo r=bird: WTF is hit?!? */
1388	/Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? /
1389	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1390	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1391	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1392	}
1393	}
1394	else
1395	{
1396	GCPdPt = CPUMGetGuestCR3(pVCpu);
1397	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1398	}
1399
1400	/* Create a reference back to the PDPT by using the index in its shadow page. */
1401	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1402	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1403	&pShwPage);
1404	AssertRCReturn(rc, rc);
1405
1406	/* Hook it up. */
1407	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1408	}
1409	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1410
1411	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1412	return VINF_SUCCESS;
1413	}
1414
1415
1416	/**
1417	* Gets the pointer to the shadow page directory entry for an address, PAE.
1418	*
1419	* @returns Pointer to the PDE.
1420	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1421	* @param GCPtr The address.
1422	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1423	*/
1424	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1425	{
1426	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1427	PGM_LOCK_ASSERT_OWNER(pVM);
1428
1429	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1430	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1431	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1432	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1433	if (!(uPdpe & X86_PDPE_P))
1434	{
1435	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1436	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1437	}
1438	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1439
1440	/* Fetch the pgm pool shadow descriptor. */
1441	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1442	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1443
1444	*ppShwPde = pShwPde;
1445	return VINF_SUCCESS;
1446	}
1447
1448
1449	/**
1450	* Syncs the SHADOW page directory pointer for the specified address.
1451	*
1452	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1453	*
1454	* The caller is responsible for making sure the guest has a valid PD before
1455	* calling this function.
1456	*
1457	* @returns VBox status code.
1458	* @param pVCpu The cross context virtual CPU structure.
1459	* @param GCPtr The address.
1460	* @param uGstPml4e Guest PML4 entry (valid).
1461	* @param uGstPdpe Guest PDPT entry (valid).
1462	* @param ppPD Receives address of page directory
1463	*/
1464	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1465	{
1466	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1467	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1468	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1469	int rc;
1470
1471	PGM_LOCK_ASSERT_OWNER(pVM);
1472
1473	/*
1474	* PML4.
1475	*/
1476	PPGMPOOLPAGE pShwPage;
1477	{
1478	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1479	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1480	X86PGPAEUINT const uPml4e = pPml4e->u;
1481
1482	/* Allocate page directory pointer table if not present. */
1483	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1484	{
1485	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1486	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1487
1488	pgmPoolCacheUsed(pPool, pShwPage);
1489
1490	/* Update the entry if needed. */
1491	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1492	\| (uPml4e & PGM_PML4_FLAGS);
1493	if (uPml4e == uPml4eNew)
1494	{ /* likely */ }
1495	else
1496	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1497	}
1498	else
1499	{
1500	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1501
1502	RTGCPTR64 GCPml4;
1503	PGMPOOLKIND enmKind;
1504	if (fNestedPagingOrNoGstPaging)
1505	{
1506	/* AMD-V nested paging or real/protected mode without paging */
1507	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1508	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1509	}
1510	else
1511	{
1512	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1513	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1514	}
1515
1516	/* Create a reference back to the PDPT by using the index in its shadow page. */
1517	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1518	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1519	&pShwPage);
1520	AssertRCReturn(rc, rc);
1521
1522	/* Hook it up. */
1523	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1524	\| (uPml4e & PGM_PML4_FLAGS));
1525	}
1526	}
1527
1528	/*
1529	* PDPT.
1530	*/
1531	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1532	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1533	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1534	X86PGPAEUINT const uPdpe = pPdpe->u;
1535
1536	/* Allocate page directory if not present. */
1537	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1538	{
1539	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1540	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1541
1542	pgmPoolCacheUsed(pPool, pShwPage);
1543
1544	/* Update the entry if needed. */
1545	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1546	\| (uPdpe & PGM_PDPT_FLAGS);
1547	if (uPdpe == uPdpeNew)
1548	{ /* likely */ }
1549	else
1550	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1551	}
1552	else
1553	{
1554	RTGCPTR64 GCPdPt;
1555	PGMPOOLKIND enmKind;
1556	if (fNestedPagingOrNoGstPaging)
1557	{
1558	/* AMD-V nested paging or real/protected mode without paging */
1559	GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1560	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1561	}
1562	else
1563	{
1564	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1565	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1566	}
1567
1568	/* Create a reference back to the PDPT by using the index in its shadow page. */
1569	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1570	pShwPage->idx, iPdPt, false /fLockPage/,
1571	&pShwPage);
1572	AssertRCReturn(rc, rc);
1573
1574	/* Hook it up. */
1575	ASMAtomicWriteU64(&pPdpe->u,
1576	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1577	}
1578
1579	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1580	return VINF_SUCCESS;
1581	}
1582
1583
1584	/**
1585	* Gets the SHADOW page directory pointer for the specified address (long mode).
1586	*
1587	* @returns VBox status code.
1588	* @param pVCpu The cross context virtual CPU structure.
1589	* @param GCPtr The address.
1590	* @param ppPml4e Receives the address of the page map level 4 entry.
1591	* @param ppPdpt Receives the address of the page directory pointer table.
1592	* @param ppPD Receives the address of the page directory.
1593	*/
1594	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1595	{
1596	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1597	PGM_LOCK_ASSERT_OWNER(pVM);
1598
1599	/*
1600	* PML4
1601	*/
1602	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1603	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1604	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1605	if (ppPml4e)
1606	*ppPml4e = (PX86PML4E)pPml4e;
1607	X86PGPAEUINT const uPml4e = pPml4e->u;
1608	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1609	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1610	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1611
1612	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1613	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1614	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1615
1616	/*
1617	* PDPT
1618	*/
1619	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1620	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1621	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1622	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1623	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1624
1625	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1626	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1627
1628	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1629	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1630	return VINF_SUCCESS;
1631	}
1632
1633
1634	/**
1635	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1636	* backing pages in case the PDPT or PML4 entry is missing.
1637	*
1638	* @returns VBox status code.
1639	* @param pVCpu The cross context virtual CPU structure.
1640	* @param GCPtr The address.
1641	* @param ppPdpt Receives address of pdpt
1642	* @param ppPD Receives address of page directory
1643	*/
1644	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1645	{
1646	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1647	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1648	int rc;
1649
1650	Assert(pVM->pgm.s.fNestedPaging);
1651	PGM_LOCK_ASSERT_OWNER(pVM);
1652
1653	/*
1654	* PML4 level.
1655	*/
1656	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1657	Assert(pPml4);
1658
1659	/* Allocate page directory pointer table if not present. */
1660	PPGMPOOLPAGE pShwPage;
1661	{
1662	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1663	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1664	EPTPML4E Pml4e;
1665	Pml4e.u = pPml4e->u;
1666	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1667	{
1668	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1669	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1670	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1671	&pShwPage);
1672	AssertRCReturn(rc, rc);
1673
1674	/* Hook up the new PDPT now. */
1675	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1676	}
1677	else
1678	{
1679	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1680	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1681
1682	pgmPoolCacheUsed(pPool, pShwPage);
1683
1684	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1685	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1686	{ }
1687	else
1688	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1689	}
1690	}
1691
1692	/*
1693	* PDPT level.
1694	*/
1695	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1696	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1697	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1698
1699	if (ppPdpt)
1700	*ppPdpt = pPdpt;
1701
1702	/* Allocate page directory if not present. */
1703	EPTPDPTE Pdpe;
1704	Pdpe.u = pPdpe->u;
1705	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1706	{
1707	RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1708	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1709	pShwPage->idx, iPdPt, false /fLockPage/,
1710	&pShwPage);
1711	AssertRCReturn(rc, rc);
1712
1713	/* Hook up the new PD now. */
1714	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1715	}
1716	else
1717	{
1718	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1719	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1720
1721	pgmPoolCacheUsed(pPool, pShwPage);
1722
1723	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1724	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1725	{ }
1726	else
1727	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1728	}
1729
1730	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1731	return VINF_SUCCESS;
1732	}
1733
1734
1735	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1736	/**
1737	* Syncs the SHADOW nested-guest page directory pointer for the specified address.
1738	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1739	*
1740	* @returns VBox status code.
1741	* @param pVCpu The cross context virtual CPU structure.
1742	* @param GCPhysNested The nested-guest physical address.
1743	* @param ppPdpt Where to store the PDPT. Optional, can be NULL.
1744	* @param ppPD Where to store the PD. Optional, can be NULL.
1745	* @param pGstWalkAll The guest walk info.
1746	*/
1747	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
1748	PPGMPTWALKGST pGstWalkAll)
1749	{
1750	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1751	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1752	int rc;
1753
1754	PPGMPOOLPAGE pShwPage;
1755	Assert(pVM->pgm.s.fNestedPaging);
1756	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
1757	PGM_LOCK_ASSERT_OWNER(pVM);
1758
1759	/*
1760	* PML4 level.
1761	*/
1762	{
1763	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1764	Assert(pPml4);
1765
1766	/* Allocate page directory pointer table if not present. */
1767	{
1768	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
1769	const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1770	PEPTPML4E pPml4e = &pPml4->a[iPml4e];
1771
1772	if (!(pPml4e->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1773	{
1774	RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
1775	rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
1776	PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /fLockPage/,
1777	&pShwPage);
1778	AssertRCReturn(rc, rc);
1779
1780	/* Hook up the new PDPT now. */
1781	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1782	}
1783	else
1784	{
1785	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1786	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1787
1788	pgmPoolCacheUsed(pPool, pShwPage);
1789
1790	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1791	if (pPml4e->u != (pShwPage->Core.Key \| fShwFlags))
1792	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1793	}
1794	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1795	Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
1796	}
1797	}
1798
1799	/*
1800	* PDPT level.
1801	*/
1802	{
1803	AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
1804
1805	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1806	if (ppPdpt)
1807	*ppPdpt = pPdpt;
1808
1809	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
1810	const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1811	PEPTPDPTE pPdpte = &pPdpt->a[iPdPte];
1812
1813	if (!(pPdpte->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1814	{
1815	RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
1816	rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1817	pShwPage->idx, iPdPte, false /fLockPage/, &pShwPage);
1818	AssertRCReturn(rc, rc);
1819
1820	/* Hook up the new PD now. */
1821	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1822	}
1823	else
1824	{
1825	pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
1826	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1827
1828	pgmPoolCacheUsed(pPool, pShwPage);
1829
1830	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1831	if (pPdpte->u != (pShwPage->Core.Key \| fShwFlags))
1832	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1833	}
1834	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1835	Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
1836
1837	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1838	}
1839
1840	return VINF_SUCCESS;
1841	}
1842	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1843
1844
1845	#ifdef IN_RING0
1846	/**
1847	* Synchronizes a range of nested page table entries.
1848	*
1849	* The caller must own the PGM lock.
1850	*
1851	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1852	* @param GCPhys Where to start.
1853	* @param cPages How many pages which entries should be synced.
1854	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1855	* host paging mode for AMD-V).
1856	*/
1857	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1858	{
1859	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1860
1861	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1862	int rc;
1863	switch (enmShwPagingMode)
1864	{
1865	case PGMMODE_32_BIT:
1866	{
1867	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1868	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1869	break;
1870	}
1871
1872	case PGMMODE_PAE:
1873	case PGMMODE_PAE_NX:
1874	{
1875	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1876	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1877	break;
1878	}
1879
1880	case PGMMODE_AMD64:
1881	case PGMMODE_AMD64_NX:
1882	{
1883	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1884	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1885	break;
1886	}
1887
1888	case PGMMODE_EPT:
1889	{
1890	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1891	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1892	break;
1893	}
1894
1895	default:
1896	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1897	}
1898	return rc;
1899	}
1900	#endif /* IN_RING0 */
1901
1902
1903	/**
1904	* Gets effective Guest OS page information.
1905	*
1906	* When GCPtr is in a big page, the function will return as if it was a normal
1907	* 4KB page. If the need for distinguishing between big and normal page becomes
1908	* necessary at a later point, a PGMGstGetPage() will be created for that
1909	* purpose.
1910	*
1911	* @returns VBox status code.
1912	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1913	* @param GCPtr Guest Context virtual address of the page.
1914	* @param pWalk Where to store the page walk information.
1915	*/
1916	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
1917	{
1918	VMCPU_ASSERT_EMT(pVCpu);
1919	Assert(pWalk);
1920	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1921	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1922	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1923	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
1924	}
1925
1926
1927	/**
1928	* Maps the guest CR3.
1929	*
1930	* @returns VBox status code.
1931	* @param pVCpu The cross context virtual CPU structure.
1932	* @param GCPhysCr3 The guest CR3 value.
1933	* @param pHCPtrGuestCr3 Where to store the mapped memory.
1934	*/
1935	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
1936	{
1937	/** @todo this needs some reworking wrt. locking? */
1938	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1939	PGM_LOCK_VOID(pVM);
1940	PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
1941	AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
1942
1943	RTHCPTR HCPtrGuestCr3;
1944	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
1945	PGM_UNLOCK(pVM);
1946
1947	*pHCPtrGuestCr3 = HCPtrGuestCr3;
1948	return rc;
1949	}
1950
1951
1952	/**
1953	* Unmaps the guest CR3.
1954	*
1955	* @returns VBox status code.
1956	* @param pVCpu The cross context virtual CPU structure.
1957	*/
1958	DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
1959	{
1960	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1961	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1962	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
1963	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
1964	}
1965
1966
1967	/**
1968	* Performs a guest page table walk.
1969	*
1970	* The guest should be in paged protect mode or long mode when making a call to
1971	* this function.
1972	*
1973	* @returns VBox status code.
1974	* @retval VINF_SUCCESS on success.
1975	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1976	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1977	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1978	*
1979	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1980	* @param GCPtr The guest virtual address to walk by.
1981	* @param pWalk Where to return the walk result. This is valid for some
1982	* error codes as well.
1983	* @param pGstWalk The guest mode specific page walk information.
1984	*/
1985	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
1986	{
1987	VMCPU_ASSERT_EMT(pVCpu);
1988	switch (pVCpu->pgm.s.enmGuestMode)
1989	{
1990	case PGMMODE_32_BIT:
1991	pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1992	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
1993
1994	case PGMMODE_PAE:
1995	case PGMMODE_PAE_NX:
1996	pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1997	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
1998
1999	case PGMMODE_AMD64:
2000	case PGMMODE_AMD64_NX:
2001	pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
2002	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
2003
2004	case PGMMODE_REAL:
2005	case PGMMODE_PROTECTED:
2006	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2007	return VERR_PGM_NOT_USED_IN_MODE;
2008
2009	case PGMMODE_EPT:
2010	case PGMMODE_NESTED_32BIT:
2011	case PGMMODE_NESTED_PAE:
2012	case PGMMODE_NESTED_AMD64:
2013	default:
2014	AssertFailed();
2015	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2016	return VERR_PGM_NOT_USED_IN_MODE;
2017	}
2018	}
2019
2020
2021	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2022	/**
2023	* Performs a guest second-level address translation (SLAT).
2024	*
2025	* @returns VBox status code.
2026	* @retval VINF_SUCCESS on success.
2027	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2028	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2029	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2030	*
2031	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2032	* @param GCPhysNested The nested-guest physical address being translated.
2033	* @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the
2034	* cause for this translation.
2035	* @param GCPtrNested The nested-guest virtual address that initiated the
2036	* SLAT. If none, pass 0 (and not NIL_RTGCPTR).
2037	* @param pWalk Where to return the walk result. This is updated for
2038	* all error codes other than
2039	* VERR_PGM_NOT_USED_IN_MODE.
2040	* @param pGstWalk Where to store the second-level paging-mode specific
2041	* walk info.
2042	*/
2043	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
2044	PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2045	{
2046	/* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
2047	Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
2048	&& pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
2049	AssertPtr(pWalk);
2050	AssertPtr(pGstWalk);
2051	switch (pVCpu->pgm.s.enmGuestSlatMode)
2052	{
2053	case PGMSLAT_EPT:
2054	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
2055	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
2056
2057	default:
2058	AssertFailed();
2059	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2060	return VERR_PGM_NOT_USED_IN_MODE;
2061	}
2062	}
2063	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
2064
2065
2066	/**
2067	* Tries to continue the previous walk.
2068	*
2069	* @note Requires the caller to hold the PGM lock from the first
2070	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
2071	* we cannot use the pointers.
2072	*
2073	* @returns VBox status code.
2074	* @retval VINF_SUCCESS on success.
2075	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2076	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2077	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2078	*
2079	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2080	* @param GCPtr The guest virtual address to walk by.
2081	* @param pWalk Pointer to the previous walk result and where to return
2082	* the result of this walk. This is valid for some error
2083	* codes as well.
2084	* @param pGstWalk The guest-mode specific walk information.
2085	*/
2086	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2087	{
2088	/*
2089	* We can only handle successfully walks.
2090	* We also limit ourselves to the next page.
2091	*/
2092	if ( pWalk->fSucceeded
2093	&& GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
2094	{
2095	Assert(pWalk->uLevel == 0);
2096	if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2097	{
2098	/*
2099	* AMD64
2100	*/
2101	if (!pWalk->fGigantPage && !pWalk->fBigPage)
2102	{
2103	/*
2104	* We fall back to full walk if the PDE table changes, if any
2105	* reserved bits are set, or if the effective page access changes.
2106	*/
2107	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
2108	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
2109	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
2110	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
2111
2112	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
2113	{
2114	if (pGstWalk->u.Amd64.pPte)
2115	{
2116	X86PTEPAE Pte;
2117	Pte.u = pGstWalk->u.Amd64.pPte[1].u;
2118	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2119	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2120	{
2121	pWalk->GCPtr = GCPtr;
2122	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2123	pGstWalk->u.Amd64.Pte.u = Pte.u;
2124	pGstWalk->u.Amd64.pPte++;
2125	return VINF_SUCCESS;
2126	}
2127	}
2128	}
2129	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
2130	{
2131	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2132	if (pGstWalk->u.Amd64.pPde)
2133	{
2134	X86PDEPAE Pde;
2135	Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2136	if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2137	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2138	{
2139	/* Get the new PTE and check out the first entry. */
2140	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2141	&pGstWalk->u.Amd64.pPt);
2142	if (RT_SUCCESS(rc))
2143	{
2144	pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2145	X86PTEPAE Pte;
2146	Pte.u = pGstWalk->u.Amd64.pPte->u;
2147	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2148	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2149	{
2150	pWalk->GCPtr = GCPtr;
2151	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2152	pGstWalk->u.Amd64.Pte.u = Pte.u;
2153	pGstWalk->u.Amd64.Pde.u = Pde.u;
2154	pGstWalk->u.Amd64.pPde++;
2155	return VINF_SUCCESS;
2156	}
2157	}
2158	}
2159	}
2160	}
2161	}
2162	else if (!pWalk->fGigantPage)
2163	{
2164	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2165	{
2166	pWalk->GCPtr = GCPtr;
2167	pWalk->GCPhys += GUEST_PAGE_SIZE;
2168	return VINF_SUCCESS;
2169	}
2170	}
2171	else
2172	{
2173	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2174	{
2175	pWalk->GCPtr = GCPtr;
2176	pWalk->GCPhys += GUEST_PAGE_SIZE;
2177	return VINF_SUCCESS;
2178	}
2179	}
2180	}
2181	}
2182	/* Case we don't handle. Do full walk. */
2183	return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2184	}
2185
2186
2187	/**
2188	* Modify page flags for a range of pages in the guest's tables
2189	*
2190	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2191	*
2192	* @returns VBox status code.
2193	* @param pVCpu The cross context virtual CPU structure.
2194	* @param GCPtr Virtual address of the first page in the range.
2195	* @param cb Size (in bytes) of the range to apply the modification to.
2196	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2197	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2198	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2199	*/
2200	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2201	{
2202	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2203	VMCPU_ASSERT_EMT(pVCpu);
2204
2205	/*
2206	* Validate input.
2207	*/
2208	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2209	Assert(cb);
2210
2211	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2212
2213	/*
2214	* Adjust input.
2215	*/
2216	cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2217	cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2218	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2219
2220	/*
2221	* Call worker.
2222	*/
2223	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2224	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2225	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2226	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2227
2228	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2229	return rc;
2230	}
2231
2232
2233	/**
2234	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2235	*
2236	* @returns @c true if the PDPE is valid, @c false otherwise.
2237	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2238	* @param paPaePdpes The PAE PDPEs to validate.
2239	*
2240	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2241	*/
2242	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2243	{
2244	Assert(paPaePdpes);
2245	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2246	{
2247	X86PDPE const PaePdpe = paPaePdpes[i];
2248	if ( !(PaePdpe.u & X86_PDPE_P)
2249	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2250	{ /* likely */ }
2251	else
2252	return false;
2253	}
2254	return true;
2255	}
2256
2257
2258	/**
2259	* Performs the lazy mapping of the 32-bit guest PD.
2260	*
2261	* @returns VBox status code.
2262	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2263	* @param ppPd Where to return the pointer to the mapping. This is
2264	* always set.
2265	*/
2266	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2267	{
2268	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2269	PGM_LOCK_VOID(pVM);
2270
2271	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2272
2273	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2274	PPGMPAGE pPage;
2275	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2276	if (RT_SUCCESS(rc))
2277	{
2278	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2279	if (RT_SUCCESS(rc))
2280	{
2281	# ifdef IN_RING3
2282	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2283	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2284	# else
2285	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2286	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2287	# endif
2288	PGM_UNLOCK(pVM);
2289	return VINF_SUCCESS;
2290	}
2291	AssertRC(rc);
2292	}
2293	PGM_UNLOCK(pVM);
2294
2295	*ppPd = NULL;
2296	return rc;
2297	}
2298
2299
2300	/**
2301	* Performs the lazy mapping of the PAE guest PDPT.
2302	*
2303	* @returns VBox status code.
2304	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2305	* @param ppPdpt Where to return the pointer to the mapping. This is
2306	* always set.
2307	*/
2308	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2309	{
2310	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2311	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2312	PGM_LOCK_VOID(pVM);
2313
2314	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2315	PPGMPAGE pPage;
2316	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2317	if (RT_SUCCESS(rc))
2318	{
2319	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2320	if (RT_SUCCESS(rc))
2321	{
2322	# ifdef IN_RING3
2323	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2324	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2325	# else
2326	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2327	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2328	# endif
2329	PGM_UNLOCK(pVM);
2330	return VINF_SUCCESS;
2331	}
2332	AssertRC(rc);
2333	}
2334
2335	PGM_UNLOCK(pVM);
2336	*ppPdpt = NULL;
2337	return rc;
2338	}
2339
2340
2341	/**
2342	* Performs the lazy mapping / updating of a PAE guest PD.
2343	*
2344	* @returns Pointer to the mapping.
2345	* @returns VBox status code.
2346	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2347	* @param iPdpt Which PD entry to map (0..3).
2348	* @param ppPd Where to return the pointer to the mapping. This is
2349	* always set.
2350	*/
2351	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2352	{
2353	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2354	PGM_LOCK_VOID(pVM);
2355
2356	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2357	Assert(pGuestPDPT);
2358	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2359	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2360	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2361
2362	PPGMPAGE pPage;
2363	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2364	if (RT_SUCCESS(rc))
2365	{
2366	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2367	AssertRC(rc);
2368	if (RT_SUCCESS(rc))
2369	{
2370	# ifdef IN_RING3
2371	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2372	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2373	# else
2374	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2375	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2376	# endif
2377	if (fChanged)
2378	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2379	PGM_UNLOCK(pVM);
2380	return VINF_SUCCESS;
2381	}
2382	}
2383
2384	/* Invalid page or some failure, invalidate the entry. */
2385	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2386	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2387	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2388
2389	PGM_UNLOCK(pVM);
2390	return rc;
2391	}
2392
2393
2394	/**
2395	* Performs the lazy mapping of the 32-bit guest PD.
2396	*
2397	* @returns VBox status code.
2398	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2399	* @param ppPml4 Where to return the pointer to the mapping. This will
2400	* always be set.
2401	*/
2402	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2403	{
2404	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2405	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2406	PGM_LOCK_VOID(pVM);
2407
2408	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2409	PPGMPAGE pPage;
2410	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2411	if (RT_SUCCESS(rc))
2412	{
2413	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2414	if (RT_SUCCESS(rc))
2415	{
2416	# ifdef IN_RING3
2417	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2418	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2419	# else
2420	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2421	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2422	# endif
2423	PGM_UNLOCK(pVM);
2424	return VINF_SUCCESS;
2425	}
2426	}
2427
2428	PGM_UNLOCK(pVM);
2429	*ppPml4 = NULL;
2430	return rc;
2431	}
2432
2433
2434	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2435	/**
2436	* Performs the lazy mapping of the guest PML4 table when using EPT paging.
2437	*
2438	* @returns VBox status code.
2439	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2440	* @param ppEptPml4 Where to return the pointer to the mapping. This will
2441	* always be set.
2442	*/
2443	int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2444	{
2445	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2446	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2447	PGM_LOCK_VOID(pVM);
2448
2449	RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2450	PPGMPAGE pPage;
2451	int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2452	if (RT_SUCCESS(rc))
2453	{
2454	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2455	if (RT_SUCCESS(rc))
2456	{
2457	# ifdef IN_RING3
2458	pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2459	pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2460	# else
2461	pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2462	pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2463	# endif
2464	PGM_UNLOCK(pVM);
2465	return VINF_SUCCESS;
2466	}
2467	}
2468
2469	PGM_UNLOCK(pVM);
2470	*ppEptPml4 = NULL;
2471	return rc;
2472	}
2473	#endif
2474
2475
2476	/**
2477	* Gets the current CR3 register value for the shadow memory context.
2478	* @returns CR3 value.
2479	* @param pVCpu The cross context virtual CPU structure.
2480	*/
2481	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2482	{
2483	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2484	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2485	return pPoolPage->Core.Key;
2486	}
2487
2488
2489	/**
2490	* Forces lazy remapping of the guest's PAE page-directory structures.
2491	*
2492	* @param pVCpu The cross context virtual CPU structure.
2493	*/
2494	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2495	{
2496	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2497	{
2498	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2499	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2500	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2501	}
2502	}
2503
2504
2505	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2506	/**
2507	* Performs second-level address translation for the given CR3 and updates the
2508	* nested-guest CR3 when successful.
2509	*
2510	* @returns VBox status code.
2511	* @param pVCpu The cross context virtual CPU structure.
2512	* @param uCr3 The masked nested-guest CR3 value.
2513	* @param pGCPhysCR3 Where to store the translated CR3.
2514	*
2515	* @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2516	* mindful of this in code that's hyper sensitive to the order of
2517	* operations.
2518	*/
2519	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2520	{
2521	if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2522	{
2523	PGMPTWALK Walk;
2524	PGMPTWALKGST GstWalk;
2525	int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk, &GstWalk);
2526	if (RT_SUCCESS(rc))
2527	{
2528	/* Update nested-guest CR3. */
2529	pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2530
2531	/* Pass back the translated result. */
2532	*pGCPhysCr3 = Walk.GCPhys;
2533	return VINF_SUCCESS;
2534	}
2535
2536	/* Translation failed. */
2537	*pGCPhysCr3 = NIL_RTGCPHYS;
2538	return rc;
2539	}
2540
2541	/*
2542	* If the nested-guest CR3 has not changed, then the previously
2543	* translated CR3 result (i.e. GCPhysCR3) is passed back.
2544	*/
2545	*pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2546	return VINF_SUCCESS;
2547	}
2548	#endif
2549
2550
2551	/**
2552	* Performs and schedules necessary updates following a CR3 load or reload.
2553	*
2554	* This will normally involve mapping the guest PD or nPDPT
2555	*
2556	* @returns VBox status code.
2557	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2558	* safely be ignored and overridden since the FF will be set too then.
2559	* @param pVCpu The cross context virtual CPU structure.
2560	* @param cr3 The new cr3.
2561	* @param fGlobal Indicates whether this is a global flush or not.
2562	*/
2563	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2564	{
2565	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2566	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2567
2568	VMCPU_ASSERT_EMT(pVCpu);
2569
2570	/*
2571	* Always flag the necessary updates; necessary for hardware acceleration
2572	*/
2573	/** @todo optimize this, it shouldn't always be necessary. */
2574	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2575	if (fGlobal)
2576	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2577
2578	/*
2579	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2580	*/
2581	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2582	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2583	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2584	if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2585	&& PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2586	{
2587	RTGCPHYS GCPhysOut;
2588	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2589	if (RT_SUCCESS(rc))
2590	GCPhysCR3 = GCPhysOut;
2591	else
2592	{
2593	/* CR3 SLAT translation failed but we try to pretend it
2594	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2595	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2596	int const rc2 = pgmGstUnmapCr3(pVCpu);
2597	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2598	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2599	return rc2;
2600	}
2601	}
2602	#endif
2603
2604	LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2605	int rc = VINF_SUCCESS;
2606	if (GCPhysOldCR3 != GCPhysCR3)
2607	{
2608	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2609	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2610	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2611
2612	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2613	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2614	if (RT_LIKELY(rc == VINF_SUCCESS))
2615	{ }
2616	else
2617	{
2618	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2619	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2620	pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2621	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2622	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2623	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2624	}
2625
2626	if (fGlobal)
2627	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2628	else
2629	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2630	}
2631	else
2632	{
2633	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2634	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2635	if (pPool->cDirtyPages)
2636	{
2637	PGM_LOCK_VOID(pVM);
2638	pgmPoolResetDirtyPages(pVM);
2639	PGM_UNLOCK(pVM);
2640	}
2641	#endif
2642	if (fGlobal)
2643	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2644	else
2645	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2646
2647	/*
2648	* Flush PAE PDPTEs.
2649	*/
2650	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2651	pgmGstFlushPaePdpes(pVCpu);
2652	}
2653
2654	IEMTlbInvalidateAll(pVCpu);
2655	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2656	return rc;
2657	}
2658
2659
2660	/**
2661	* Performs and schedules necessary updates following a CR3 load or reload when
2662	* using nested or extended paging.
2663	*
2664	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2665	* TLB and triggering a SyncCR3.
2666	*
2667	* This will normally involve mapping the guest PD or nPDPT
2668	*
2669	* @returns VBox status code.
2670	* @retval VINF_SUCCESS.
2671	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2672	* paging modes). This can safely be ignored and overridden since the
2673	* FF will be set too then.
2674	* @param pVCpu The cross context virtual CPU structure.
2675	* @param cr3 The new CR3.
2676	*/
2677	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2678	{
2679	VMCPU_ASSERT_EMT(pVCpu);
2680
2681	/* We assume we're only called in nested paging mode. */
2682	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2683
2684	/*
2685	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2686	*/
2687	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2688	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2689	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2690	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2691	{
2692	RTGCPHYS GCPhysOut;
2693	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2694	if (RT_SUCCESS(rc))
2695	GCPhysCR3 = GCPhysOut;
2696	else
2697	{
2698	/* CR3 SLAT translation failed but we try to pretend it
2699	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2700	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2701	int const rc2 = pgmGstUnmapCr3(pVCpu);
2702	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2703	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2704	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2705	return rc2;
2706	}
2707	}
2708	#endif
2709
2710	LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2711	int rc = VINF_SUCCESS;
2712	if (GCPhysOldCR3 != GCPhysCR3)
2713	{
2714	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2715	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2716	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2717
2718	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2719	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2720
2721	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2722	}
2723	/*
2724	* Flush PAE PDPTEs.
2725	*/
2726	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2727	pgmGstFlushPaePdpes(pVCpu);
2728
2729	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2730	return rc;
2731	}
2732
2733
2734	/**
2735	* Synchronize the paging structures.
2736	*
2737	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2738	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2739	* in several places, most importantly whenever the CR3 is loaded.
2740	*
2741	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2742	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2743	* the VMM into guest context.
2744	* @param pVCpu The cross context virtual CPU structure.
2745	* @param cr0 Guest context CR0 register
2746	* @param cr3 Guest context CR3 register
2747	* @param cr4 Guest context CR4 register
2748	* @param fGlobal Including global page directories or not
2749	*/
2750	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2751	{
2752	int rc;
2753
2754	VMCPU_ASSERT_EMT(pVCpu);
2755
2756	/*
2757	* The pool may have pending stuff and even require a return to ring-3 to
2758	* clear the whole thing.
2759	*/
2760	rc = pgmPoolSyncCR3(pVCpu);
2761	if (rc != VINF_SUCCESS)
2762	return rc;
2763
2764	/*
2765	* We might be called when we shouldn't.
2766	*
2767	* The mode switching will ensure that the PD is resynced after every mode
2768	* switch. So, if we find ourselves here when in protected or real mode
2769	* we can safely clear the FF and return immediately.
2770	*/
2771	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2772	{
2773	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2774	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2775	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2776	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2777	return VINF_SUCCESS;
2778	}
2779
2780	/* If global pages are not supported, then all flushes are global. */
2781	if (!(cr4 & X86_CR4_PGE))
2782	fGlobal = true;
2783	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2784	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2785
2786	/*
2787	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2788	* This should be done before SyncCR3.
2789	*/
2790	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2791	{
2792	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2793
2794	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2795	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2796	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2797	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2798	{
2799	RTGCPHYS GCPhysOut;
2800	int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2801	if (RT_SUCCESS(rc2))
2802	GCPhysCR3 = GCPhysOut;
2803	else
2804	{
2805	/* CR3 SLAT translation failed but we try to pretend it
2806	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2807	AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2));
2808	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2809	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2810	return rc2;
2811	}
2812	}
2813	#endif
2814	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2815	if (GCPhysOldCR3 != GCPhysCR3)
2816	{
2817	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2818	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2819	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2820	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2821	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2822	}
2823
2824	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2825	if ( rc == VINF_PGM_SYNC_CR3
2826	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2827	{
2828	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2829	#ifdef IN_RING3
2830	rc = pgmPoolSyncCR3(pVCpu);
2831	#else
2832	if (rc == VINF_PGM_SYNC_CR3)
2833	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2834	return VINF_PGM_SYNC_CR3;
2835	#endif
2836	}
2837	AssertRCReturn(rc, rc);
2838	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2839	}
2840
2841	/*
2842	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2843	*/
2844	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2845
2846	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2847	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2848	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2849	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2850
2851	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2852	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2853	if (rc == VINF_SUCCESS)
2854	{
2855	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2856	{
2857	/* Go back to ring 3 if a pgm pool sync is again pending. */
2858	return VINF_PGM_SYNC_CR3;
2859	}
2860
2861	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2862	{
2863	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2864	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2865	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2866	}
2867	}
2868
2869	/*
2870	* Now flush the CR3 (guest context).
2871	*/
2872	if (rc == VINF_SUCCESS)
2873	PGM_INVL_VCPU_TLBS(pVCpu);
2874	return rc;
2875	}
2876
2877
2878	/**
2879	* Maps all the PAE PDPE entries.
2880	*
2881	* @returns VBox status code.
2882	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2883	* @param paPaePdpes The new PAE PDPE values.
2884	*
2885	* @remarks This function may be invoked during the process of changing the guest
2886	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
2887	* reflect PAE paging just yet.
2888	*/
2889	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2890	{
2891	Assert(paPaePdpes);
2892	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2893	{
2894	X86PDPE const PaePdpe = paPaePdpes[i];
2895
2896	/*
2897	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
2898	* are deferred.[1] Also, different situations require different handling of invalid
2899	* PDPE entries. Here we assume the caller has already validated or doesn't require
2900	* validation of the PDPEs.
2901	*
2902	* In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
2903	* validated by the VMX transition.
2904	*
2905	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
2906	*/
2907	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
2908	{
2909	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2910	RTHCPTR HCPtr;
2911
2912	RTGCPHYS GCPhys;
2913	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2914	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2915	{
2916	PGMPTWALK Walk;
2917	PGMPTWALKGST GstWalk;
2918	RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
2919	int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid /, 0 / GCPtrNested */,
2920	&Walk, &GstWalk);
2921	if (RT_SUCCESS(rc))
2922	GCPhys = Walk.GCPhys;
2923	else
2924	{
2925	/*
2926	* Second-level address translation of the PAE PDPE has failed but we must -NOT-
2927	* abort and return a failure now. This is because we're called from a Mov CRx
2928	* instruction (or similar operation). Let's just pretend success but flag that
2929	* we need to map this PDPE lazily later.
2930	*
2931	* See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
2932	* See Intel spec. 28.3.1 "EPT Overview".
2933	*/
2934	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2935	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2936	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2937	continue;
2938	}
2939	}
2940	else
2941	#endif
2942	{
2943	GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
2944	}
2945
2946	PGM_LOCK_VOID(pVM);
2947	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2948	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
2949	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
2950	PGM_UNLOCK(pVM);
2951	if (RT_SUCCESS(rc))
2952	{
2953	#ifdef IN_RING3
2954	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
2955	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
2956	#else
2957	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
2958	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
2959	#endif
2960	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
2961	continue;
2962	}
2963	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
2964	}
2965	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2966	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2967	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2968	}
2969	return VINF_SUCCESS;
2970	}
2971
2972
2973	/**
2974	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
2975	*
2976	* @returns VBox status code.
2977	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2978	* @param cr3 The guest CR3 value.
2979	*
2980	* @remarks This function may be invoked during the process of changing the guest
2981	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
2982	* PAE paging just yet.
2983	*/
2984	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
2985	{
2986	/*
2987	* Read the page-directory-pointer table (PDPT) at CR3.
2988	*/
2989	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
2990	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2991
2992	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2993	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2994	{
2995	RTGCPHYS GCPhysOut;
2996	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2997	if (RT_SUCCESS(rc))
2998	GCPhysCR3 = GCPhysOut;
2999	else
3000	{
3001	Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
3002	return rc;
3003	}
3004	}
3005	#endif
3006
3007	RTHCPTR HCPtrGuestCr3;
3008	int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
3009	if (RT_SUCCESS(rc))
3010	{
3011	/*
3012	* Validate the page-directory-pointer table entries (PDPE).
3013	*/
3014	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
3015	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
3016	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
3017	{
3018	/*
3019	* Map the PDPT.
3020	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
3021	* that PGMFlushTLB will be called soon and only a change to CR3 then
3022	* will cause the shadow page tables to be updated.
3023	*/
3024	#ifdef IN_RING3
3025	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
3026	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
3027	#else
3028	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
3029	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
3030	#endif
3031
3032	/*
3033	* Update CPUM and map the 4 PAE PDPEs.
3034	*/
3035	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
3036	rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
3037	if (RT_SUCCESS(rc))
3038	{
3039	#ifdef IN_RING3
3040	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
3041	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
3042	#else
3043	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
3044	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
3045	#endif
3046	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
3047	}
3048	}
3049	else
3050	rc = VERR_PGM_PAE_PDPE_RSVD;
3051	}
3052	return rc;
3053	}
3054
3055
3056	/**
3057	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
3058	*
3059	* @returns VBox status code, with the following informational code for
3060	* VM scheduling.
3061	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
3062	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
3063	*
3064	* @param pVCpu The cross context virtual CPU structure.
3065	* @param cr0 The new cr0.
3066	* @param cr4 The new cr4.
3067	* @param efer The new extended feature enable register.
3068	* @param fForce Whether to force a mode change.
3069	*/
3070	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
3071	{
3072	VMCPU_ASSERT_EMT(pVCpu);
3073
3074	/*
3075	* Calc the new guest mode.
3076	*
3077	* Note! We check PG before PE and without requiring PE because of the
3078	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
3079	*/
3080	PGMMODE enmGuestMode;
3081	if (cr0 & X86_CR0_PG)
3082	{
3083	if (!(cr4 & X86_CR4_PAE))
3084	{
3085	bool const fPse = !!(cr4 & X86_CR4_PSE);
3086	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
3087	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
3088	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
3089	enmGuestMode = PGMMODE_32_BIT;
3090	}
3091	else if (!(efer & MSR_K6_EFER_LME))
3092	{
3093	if (!(efer & MSR_K6_EFER_NXE))
3094	enmGuestMode = PGMMODE_PAE;
3095	else
3096	enmGuestMode = PGMMODE_PAE_NX;
3097	}
3098	else
3099	{
3100	if (!(efer & MSR_K6_EFER_NXE))
3101	enmGuestMode = PGMMODE_AMD64;
3102	else
3103	enmGuestMode = PGMMODE_AMD64_NX;
3104	}
3105	}
3106	else if (!(cr0 & X86_CR0_PE))
3107	enmGuestMode = PGMMODE_REAL;
3108	else
3109	enmGuestMode = PGMMODE_PROTECTED;
3110
3111	/*
3112	* Did it change?
3113	*/
3114	if ( !fForce
3115	&& pVCpu->pgm.s.enmGuestMode == enmGuestMode)
3116	return VINF_SUCCESS;
3117
3118	/* Flush the TLB */
3119	PGM_INVL_VCPU_TLBS(pVCpu);
3120	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3121	}
3122
3123
3124	/**
3125	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3126	*
3127	* @returns PGM_TYPE_*.
3128	* @param pgmMode The mode value to convert.
3129	*/
3130	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3131	{
3132	switch (pgmMode)
3133	{
3134	case PGMMODE_REAL: return PGM_TYPE_REAL;
3135	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3136	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3137	case PGMMODE_PAE:
3138	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3139	case PGMMODE_AMD64:
3140	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3141	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3142	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3143	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3144	case PGMMODE_EPT: return PGM_TYPE_EPT;
3145	case PGMMODE_NONE: return PGM_TYPE_NONE;
3146	default:
3147	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3148	}
3149	}
3150
3151
3152	/**
3153	* Calculates the shadow paging mode.
3154	*
3155	* @returns The shadow paging mode.
3156	* @param pVM The cross context VM structure.
3157	* @param enmGuestMode The guest mode.
3158	* @param enmHostMode The host mode.
3159	* @param enmShadowMode The current shadow mode.
3160	*/
3161	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3162	{
3163	switch (enmGuestMode)
3164	{
3165	case PGMMODE_REAL:
3166	case PGMMODE_PROTECTED:
3167	switch (enmHostMode)
3168	{
3169	case SUPPAGINGMODE_32_BIT:
3170	case SUPPAGINGMODE_32_BIT_GLOBAL:
3171	enmShadowMode = PGMMODE_32_BIT;
3172	break;
3173
3174	case SUPPAGINGMODE_PAE:
3175	case SUPPAGINGMODE_PAE_NX:
3176	case SUPPAGINGMODE_PAE_GLOBAL:
3177	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3178	enmShadowMode = PGMMODE_PAE;
3179	break;
3180
3181	case SUPPAGINGMODE_AMD64:
3182	case SUPPAGINGMODE_AMD64_GLOBAL:
3183	case SUPPAGINGMODE_AMD64_NX:
3184	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3185	enmShadowMode = PGMMODE_PAE;
3186	break;
3187
3188	default:
3189	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3190	}
3191	break;
3192
3193	case PGMMODE_32_BIT:
3194	switch (enmHostMode)
3195	{
3196	case SUPPAGINGMODE_32_BIT:
3197	case SUPPAGINGMODE_32_BIT_GLOBAL:
3198	enmShadowMode = PGMMODE_32_BIT;
3199	break;
3200
3201	case SUPPAGINGMODE_PAE:
3202	case SUPPAGINGMODE_PAE_NX:
3203	case SUPPAGINGMODE_PAE_GLOBAL:
3204	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3205	enmShadowMode = PGMMODE_PAE;
3206	break;
3207
3208	case SUPPAGINGMODE_AMD64:
3209	case SUPPAGINGMODE_AMD64_GLOBAL:
3210	case SUPPAGINGMODE_AMD64_NX:
3211	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3212	enmShadowMode = PGMMODE_PAE;
3213	break;
3214
3215	default:
3216	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3217	}
3218	break;
3219
3220	case PGMMODE_PAE:
3221	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3222	switch (enmHostMode)
3223	{
3224	case SUPPAGINGMODE_32_BIT:
3225	case SUPPAGINGMODE_32_BIT_GLOBAL:
3226	enmShadowMode = PGMMODE_PAE;
3227	break;
3228
3229	case SUPPAGINGMODE_PAE:
3230	case SUPPAGINGMODE_PAE_NX:
3231	case SUPPAGINGMODE_PAE_GLOBAL:
3232	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3233	enmShadowMode = PGMMODE_PAE;
3234	break;
3235
3236	case SUPPAGINGMODE_AMD64:
3237	case SUPPAGINGMODE_AMD64_GLOBAL:
3238	case SUPPAGINGMODE_AMD64_NX:
3239	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3240	enmShadowMode = PGMMODE_PAE;
3241	break;
3242
3243	default:
3244	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3245	}
3246	break;
3247
3248	case PGMMODE_AMD64:
3249	case PGMMODE_AMD64_NX:
3250	switch (enmHostMode)
3251	{
3252	case SUPPAGINGMODE_32_BIT:
3253	case SUPPAGINGMODE_32_BIT_GLOBAL:
3254	enmShadowMode = PGMMODE_AMD64;
3255	break;
3256
3257	case SUPPAGINGMODE_PAE:
3258	case SUPPAGINGMODE_PAE_NX:
3259	case SUPPAGINGMODE_PAE_GLOBAL:
3260	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3261	enmShadowMode = PGMMODE_AMD64;
3262	break;
3263
3264	case SUPPAGINGMODE_AMD64:
3265	case SUPPAGINGMODE_AMD64_GLOBAL:
3266	case SUPPAGINGMODE_AMD64_NX:
3267	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3268	enmShadowMode = PGMMODE_AMD64;
3269	break;
3270
3271	default:
3272	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3273	}
3274	break;
3275
3276	default:
3277	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3278	}
3279
3280	/*
3281	* Override the shadow mode when NEM, IEM or nested paging is active.
3282	*/
3283	if (!VM_IS_HM_ENABLED(pVM))
3284	{
3285	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
3286	pVM->pgm.s.fNestedPaging = true;
3287	enmShadowMode = PGMMODE_NONE;
3288	}
3289	else
3290	{
3291	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3292	pVM->pgm.s.fNestedPaging = fNestedPaging;
3293	if (fNestedPaging)
3294	{
3295	if (HMIsVmxActive(pVM))
3296	enmShadowMode = PGMMODE_EPT;
3297	else
3298	{
3299	/* The nested SVM paging depends on the host one. */
3300	Assert(HMIsSvmActive(pVM));
3301	if ( enmGuestMode == PGMMODE_AMD64
3302	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3303	enmShadowMode = PGMMODE_NESTED_AMD64;
3304	else
3305	switch (pVM->pgm.s.enmHostMode)
3306	{
3307	case SUPPAGINGMODE_32_BIT:
3308	case SUPPAGINGMODE_32_BIT_GLOBAL:
3309	enmShadowMode = PGMMODE_NESTED_32BIT;
3310	break;
3311
3312	case SUPPAGINGMODE_PAE:
3313	case SUPPAGINGMODE_PAE_GLOBAL:
3314	case SUPPAGINGMODE_PAE_NX:
3315	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3316	enmShadowMode = PGMMODE_NESTED_PAE;
3317	break;
3318
3319	case SUPPAGINGMODE_AMD64:
3320	case SUPPAGINGMODE_AMD64_GLOBAL:
3321	case SUPPAGINGMODE_AMD64_NX:
3322	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3323	enmShadowMode = PGMMODE_NESTED_AMD64;
3324	break;
3325
3326	default:
3327	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3328	}
3329	}
3330	}
3331	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3332	else
3333	{
3334	/* Nested paging is a requirement for nested VT-x. */
3335	AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3336	}
3337	#endif
3338	}
3339
3340	return enmShadowMode;
3341	}
3342
3343
3344	/**
3345	* Performs the actual mode change.
3346	* This is called by PGMChangeMode and pgmR3InitPaging().
3347	*
3348	* @returns VBox status code. May suspend or power off the VM on error, but this
3349	* will trigger using FFs and not informational status codes.
3350	*
3351	* @param pVM The cross context VM structure.
3352	* @param pVCpu The cross context virtual CPU structure.
3353	* @param enmGuestMode The new guest mode. This is assumed to be different from
3354	* the current mode.
3355	* @param fForce Whether to force a shadow paging mode change.
3356	*/
3357	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3358	{
3359	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3360	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3361
3362	/*
3363	* Calc the shadow mode and switcher.
3364	*/
3365	PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3366	bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode \|\| fForce;
3367
3368	/*
3369	* Exit old mode(s).
3370	*/
3371	/* shadow */
3372	if (fShadowModeChanged)
3373	{
3374	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3375	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3376	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3377	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3378	{
3379	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3380	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3381	}
3382	}
3383	else
3384	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3385
3386	/* guest */
3387	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3388	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3389	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3390	{
3391	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3392	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3393	}
3394	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3395	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3396	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3397	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3398
3399	/*
3400	* Change the paging mode data indexes.
3401	*/
3402	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3403	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3404	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3405	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3406	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3407	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3408	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3409	#ifdef IN_RING3
3410	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3411	#endif
3412
3413	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3414	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3415	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3416	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3417	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3418	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3419	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3420	#ifdef IN_RING3
3421	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3422	#endif
3423
3424	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3425	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3426	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3427	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3428	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3429	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3430	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3431	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3432	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3433	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3434	#ifdef VBOX_STRICT
3435	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3436	#endif
3437
3438	/*
3439	* Determine SLAT mode -before- entering the new shadow mode!
3440	*/
3441	pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
3442
3443	/*
3444	* Enter new shadow mode (if changed).
3445	*/
3446	if (fShadowModeChanged)
3447	{
3448	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3449	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
3450	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3451	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3452	}
3453
3454	/*
3455	* Always flag the necessary updates
3456	*/
3457	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3458
3459	/*
3460	* Enter the new guest and shadow+guest modes.
3461	*/
3462	/* Calc the new CR3 value. */
3463	RTGCPHYS GCPhysCR3;
3464	switch (enmGuestMode)
3465	{
3466	case PGMMODE_REAL:
3467	case PGMMODE_PROTECTED:
3468	GCPhysCR3 = NIL_RTGCPHYS;
3469	break;
3470
3471	case PGMMODE_32_BIT:
3472	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3473	break;
3474
3475	case PGMMODE_PAE_NX:
3476	case PGMMODE_PAE:
3477	if (!pVM->cpum.ro.GuestFeatures.fPae)
3478	#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3479	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3480	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3481	#else
3482	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3483
3484	#endif
3485	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3486	break;
3487
3488	#ifdef VBOX_WITH_64_BITS_GUESTS
3489	case PGMMODE_AMD64_NX:
3490	case PGMMODE_AMD64:
3491	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3492	break;
3493	#endif
3494	default:
3495	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3496	}
3497
3498	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3499	/*
3500	* If a nested-guest is using EPT paging:
3501	* - Update the second-level address translation (SLAT) mode.
3502	* - Indicate that the CR3 is nested-guest physical address.
3503	*/
3504	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3505	{
3506	if (PGMMODE_WITH_PAGING(enmGuestMode))
3507	{
3508	/*
3509	* Translate CR3 to its guest-physical address.
3510	* We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3511	* switching modes to keep it consistent with how GCPhysCR3 is updated.
3512	*/
3513	PGMPTWALK Walk;
3514	PGMPTWALKGST GstWalk;
3515	int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk,
3516	&GstWalk);
3517	if (RT_SUCCESS(rc))
3518	{ /* likely */ }
3519	else
3520	{
3521	/*
3522	* SLAT failed but we avoid reporting this to the caller because the caller
3523	* is not supposed to fail. The only time the caller needs to indicate a
3524	* failure to software is when PAE paging is used by the nested-guest, but
3525	* we handle the PAE case separately (e.g., see VMX transition in IEM).
3526	* In all other cases, the failure will be indicated when CR3 tries to be
3527	* translated on the next linear-address memory access.
3528	* See Intel spec. 27.2.1 "EPT Overview".
3529	*/
3530	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3531
3532	/* Trying to coax PGM to succeed for the time being... */
3533	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3534	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3535	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3536	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3537	return VINF_SUCCESS;
3538	}
3539	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3540	GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
3541	}
3542	}
3543	else
3544	Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3545	#endif
3546
3547	/*
3548	* Enter the new guest mode.
3549	*/
3550	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3551	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3552	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3553
3554	/* Set the new guest CR3 (and nested-guest CR3). */
3555	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3556
3557	/* status codes. */
3558	AssertRC(rc);
3559	AssertRC(rc2);
3560	if (RT_SUCCESS(rc))
3561	{
3562	rc = rc2;
3563	if (RT_SUCCESS(rc)) /* no informational status codes. */
3564	rc = VINF_SUCCESS;
3565	}
3566
3567	/*
3568	* Notify HM.
3569	*/
3570	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3571	return rc;
3572	}
3573
3574
3575	/**
3576	* Called by CPUM or REM when CR0.WP changes to 1.
3577	*
3578	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3579	* @thread EMT
3580	*/
3581	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3582	{
3583	/*
3584	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3585	*
3586	* Use the counter to judge whether there might be pool pages with active
3587	* hacks in them. If there are, we will be running the risk of messing up
3588	* the guest by allowing it to write to read-only pages. Thus, we have to
3589	* clear the page pool ASAP if there is the slightest chance.
3590	*/
3591	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3592	{
3593	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3594
3595	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3596	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3597	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3598	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3599	}
3600	}
3601
3602
3603	/**
3604	* Gets the current guest paging mode.
3605	*
3606	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3607	*
3608	* @returns The current paging mode.
3609	* @param pVCpu The cross context virtual CPU structure.
3610	*/
3611	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3612	{
3613	return pVCpu->pgm.s.enmGuestMode;
3614	}
3615
3616
3617	/**
3618	* Gets the current shadow paging mode.
3619	*
3620	* @returns The current paging mode.
3621	* @param pVCpu The cross context virtual CPU structure.
3622	*/
3623	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3624	{
3625	return pVCpu->pgm.s.enmShadowMode;
3626	}
3627
3628
3629	/**
3630	* Gets the current host paging mode.
3631	*
3632	* @returns The current paging mode.
3633	* @param pVM The cross context VM structure.
3634	*/
3635	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3636	{
3637	switch (pVM->pgm.s.enmHostMode)
3638	{
3639	case SUPPAGINGMODE_32_BIT:
3640	case SUPPAGINGMODE_32_BIT_GLOBAL:
3641	return PGMMODE_32_BIT;
3642
3643	case SUPPAGINGMODE_PAE:
3644	case SUPPAGINGMODE_PAE_GLOBAL:
3645	return PGMMODE_PAE;
3646
3647	case SUPPAGINGMODE_PAE_NX:
3648	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3649	return PGMMODE_PAE_NX;
3650
3651	case SUPPAGINGMODE_AMD64:
3652	case SUPPAGINGMODE_AMD64_GLOBAL:
3653	return PGMMODE_AMD64;
3654
3655	case SUPPAGINGMODE_AMD64_NX:
3656	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3657	return PGMMODE_AMD64_NX;
3658
3659	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3660	}
3661
3662	return PGMMODE_INVALID;
3663	}
3664
3665
3666	/**
3667	* Get mode name.
3668	*
3669	* @returns read-only name string.
3670	* @param enmMode The mode which name is desired.
3671	*/
3672	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3673	{
3674	switch (enmMode)
3675	{
3676	case PGMMODE_REAL: return "Real";
3677	case PGMMODE_PROTECTED: return "Protected";
3678	case PGMMODE_32_BIT: return "32-bit";
3679	case PGMMODE_PAE: return "PAE";
3680	case PGMMODE_PAE_NX: return "PAE+NX";
3681	case PGMMODE_AMD64: return "AMD64";
3682	case PGMMODE_AMD64_NX: return "AMD64+NX";
3683	case PGMMODE_NESTED_32BIT: return "Nested-32";
3684	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3685	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3686	case PGMMODE_EPT: return "EPT";
3687	case PGMMODE_NONE: return "None";
3688	default: return "unknown mode value";
3689	}
3690	}
3691
3692
3693	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3694	/**
3695	* Gets the SLAT mode name.
3696	*
3697	* @returns The read-only SLAT mode descriptive string.
3698	* @param enmSlatMode The SLAT mode value.
3699	*/
3700	VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3701	{
3702	switch (enmSlatMode)
3703	{
3704	case PGMSLAT_DIRECT: return "Direct";
3705	case PGMSLAT_EPT: return "EPT";
3706	case PGMSLAT_32BIT: return "32-bit";
3707	case PGMSLAT_PAE: return "PAE";
3708	case PGMSLAT_AMD64: return "AMD64";
3709	default: return "Unknown";
3710	}
3711	}
3712	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3713
3714
3715	/**
3716	* Gets the physical address represented in the guest CR3 as PGM sees it.
3717	*
3718	* This is mainly for logging and debugging.
3719	*
3720	* @returns PGM's guest CR3 value.
3721	* @param pVCpu The cross context virtual CPU structure.
3722	*/
3723	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3724	{
3725	return pVCpu->pgm.s.GCPhysCR3;
3726	}
3727
3728
3729
3730	/**
3731	* Notification from CPUM that the EFER.NXE bit has changed.
3732	*
3733	* @param pVCpu The cross context virtual CPU structure of the CPU for
3734	* which EFER changed.
3735	* @param fNxe The new NXE state.
3736	*/
3737	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3738	{
3739	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3740	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3741
3742	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3743	if (fNxe)
3744	{
3745	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3746	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3747	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3748	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3749	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3750	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3751	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3752	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3753	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3754	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3755	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3756
3757	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3758	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3759	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3760	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3761	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3762	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3763	}
3764	else
3765	{
3766	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3767	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3768	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3769	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3770	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3771	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3772	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3773	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3774	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3775	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3776	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3777
3778	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3779	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3780	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3781	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3782	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3783	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3784	}
3785	}
3786
3787
3788	/**
3789	* Check if any pgm pool pages are marked dirty (not monitored)
3790	*
3791	* @returns bool locked/not locked
3792	* @param pVM The cross context VM structure.
3793	*/
3794	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3795	{
3796	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3797	}
3798
3799
3800	/**
3801	* Check if this VCPU currently owns the PGM lock.
3802	*
3803	* @returns bool owner/not owner
3804	* @param pVM The cross context VM structure.
3805	*/
3806	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3807	{
3808	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3809	}
3810
3811
3812	/**
3813	* Enable or disable large page usage
3814	*
3815	* @returns VBox status code.
3816	* @param pVM The cross context VM structure.
3817	* @param fUseLargePages Use/not use large pages
3818	*/
3819	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3820	{
3821	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3822
3823	pVM->pgm.s.fUseLargePages = fUseLargePages;
3824	return VINF_SUCCESS;
3825	}
3826
3827
3828	/**
3829	* Acquire the PGM lock.
3830	*
3831	* @returns VBox status code
3832	* @param pVM The cross context VM structure.
3833	* @param fVoid Set if the caller cannot handle failure returns.
3834	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3835	*/
3836	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3837	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3838	#else
3839	int pgmLock(PVMCC pVM, bool fVoid)
3840	#endif
3841	{
3842	#if defined(VBOX_STRICT)
3843	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3844	#else
3845	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3846	#endif
3847	if (RT_SUCCESS(rc))
3848	return rc;
3849	if (fVoid)
3850	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3851	else
3852	AssertRC(rc);
3853	return rc;
3854	}
3855
3856
3857	/**
3858	* Release the PGM lock.
3859	*
3860	* @returns VBox status code
3861	* @param pVM The cross context VM structure.
3862	*/
3863	void pgmUnlock(PVMCC pVM)
3864	{
3865	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3866	pVM->pgm.s.cDeprecatedPageLocks = 0;
3867	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3868	if (rc == VINF_SEM_NESTED)
3869	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3870	}
3871
3872
3873	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3874
3875	/** Format handler for PGMPAGE.
3876	* @copydoc FNRTSTRFORMATTYPE */
3877	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3878	const char pszType, void const pvValue,
3879	int cchWidth, int cchPrecision, unsigned fFlags,
3880	void *pvUser)
3881	{
3882	size_t cch;
3883	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3884	if (RT_VALID_PTR(pPage))
3885	{
3886	char szTmp[64+80];
3887
3888	cch = 0;
3889
3890	/* The single char state stuff. */
3891	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3892	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3893
3894	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3895	if (IS_PART_INCLUDED(5))
3896	{
3897	static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
3898	szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
3899	\| ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
3900	}
3901
3902	/* The type. */
3903	if (IS_PART_INCLUDED(4))
3904	{
3905	szTmp[cch++] = ':';
3906	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3907	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3908	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3909	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3910	}
3911
3912	/* The numbers. */
3913	if (IS_PART_INCLUDED(3))
3914	{
3915	szTmp[cch++] = ':';
3916	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3917	}
3918
3919	if (IS_PART_INCLUDED(2))
3920	{
3921	szTmp[cch++] = ':';
3922	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3923	}
3924
3925	if (IS_PART_INCLUDED(6))
3926	{
3927	szTmp[cch++] = ':';
3928	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3929	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3930	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3931	}
3932	# undef IS_PART_INCLUDED
3933
3934	cch = pfnOutput(pvArgOutput, szTmp, cch);
3935	}
3936	else
3937	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3938	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3939	return cch;
3940	}
3941
3942
3943	/** Format handler for PGMRAMRANGE.
3944	* @copydoc FNRTSTRFORMATTYPE */
3945	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3946	const char pszType, void const pvValue,
3947	int cchWidth, int cchPrecision, unsigned fFlags,
3948	void *pvUser)
3949	{
3950	size_t cch;
3951	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3952	if (RT_VALID_PTR(pRam))
3953	{
3954	char szTmp[80];
3955	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3956	cch = pfnOutput(pvArgOutput, szTmp, cch);
3957	}
3958	else
3959	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3960	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3961	return cch;
3962	}
3963
3964	/** Format type andlers to be registered/deregistered. */
3965	static const struct
3966	{
3967	char szType[24];
3968	PFNRTSTRFORMATTYPE pfnHandler;
3969	} g_aPgmFormatTypes[] =
3970	{
3971	{ "pgmpage", pgmFormatTypeHandlerPage },
3972	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3973	};
3974
3975	#endif /* !IN_R0 \|\| LOG_ENABLED */
3976
3977	/**
3978	* Registers the global string format types.
3979	*
3980	* This should be called at module load time or in some other manner that ensure
3981	* that it's called exactly one time.
3982	*
3983	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3984	*/
3985	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3986	{
3987	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3988	int rc = VINF_SUCCESS;
3989	unsigned i;
3990	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3991	{
3992	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3993	# ifdef IN_RING0
3994	if (rc == VERR_ALREADY_EXISTS)
3995	{
3996	/* in case of cleanup failure in ring-0 */
3997	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3998	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3999	}
4000	# endif
4001	}
4002	if (RT_FAILURE(rc))
4003	while (i-- > 0)
4004	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4005
4006	return rc;
4007	#else
4008	return VINF_SUCCESS;
4009	#endif
4010	}
4011
4012
4013	/**
4014	* Deregisters the global string format types.
4015	*
4016	* This should be called at module unload time or in some other manner that
4017	* ensure that it's called exactly one time.
4018	*/
4019	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
4020	{
4021	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4022	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4023	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4024	#endif
4025	}
4026
4027
4028	#ifdef VBOX_STRICT
4029	/**
4030	* Asserts that everything related to the guest CR3 is correctly shadowed.
4031	*
4032	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
4033	* and assert the correctness of the guest CR3 mapping before asserting that the
4034	* shadow page tables is in sync with the guest page tables.
4035	*
4036	* @returns Number of conflicts.
4037	* @param pVM The cross context VM structure.
4038	* @param pVCpu The cross context virtual CPU structure.
4039	* @param cr3 The current guest CR3 register value.
4040	* @param cr4 The current guest CR4 register value.
4041	*/
4042	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
4043	{
4044	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4045
4046	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
4047	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
4048	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
4049
4050	PGM_LOCK_VOID(pVM);
4051	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
4052	PGM_UNLOCK(pVM);
4053
4054	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4055	return cErrors;
4056	}
4057	#endif /* VBOX_STRICT */
4058
4059
4060	/**
4061	* Updates PGM's copy of the guest's EPT pointer.
4062	*
4063	* @param pVCpu The cross context virtual CPU structure.
4064	* @param uEptPtr The EPT pointer.
4065	*
4066	* @remarks This can be called as part of VM-entry so we might be in the midst of
4067	* switching to VMX non-root mode.
4068	*/
4069	VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
4070	{
4071	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4072	PGM_LOCK_VOID(pVM);
4073	pVCpu->pgm.s.uEptPtr = uEptPtr;
4074	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4075	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4076	PGM_UNLOCK(pVM);
4077	}
4078

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 99654

Download in other formats: