PGMAll.cpp@ 107631

Last change on this file since 107631 was 107171, checked in by vboxsync, 7 weeks ago
VMM/PGM: Introducing VBOX_WITH_ONLY_PGM_NEM_MODE to disable lots unused code on *.arm64 and darwin. jiraref:VBP-1466
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 153.8 KB

Line
1	/* $Id: PGMAll.cpp 107171 2024-11-28 10:38:10Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/cpum.h>
36	#include <VBox/vmm/selm.h>
37	#include <VBox/vmm/iem.h>
38	#include <VBox/vmm/iom.h>
39	#include <VBox/sup.h>
40	#include <VBox/vmm/mm.h>
41	#include <VBox/vmm/stam.h>
42	#include <VBox/vmm/trpm.h>
43	#include <VBox/vmm/em.h>
44	#include <VBox/vmm/hm.h>
45	#include <VBox/vmm/hm_vmx.h>
46	#include "PGMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include "PGMInline.h"
49	#include <iprt/assert.h>
50	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
51	# include <iprt/asm-amd64-x86.h>
52	#endif
53	#include <iprt/string.h>
54	#include <VBox/log.h>
55	#include <VBox/param.h>
56	#include <VBox/err.h>
57
58
59	/*********************************************************************************************************************************
60	* Internal Functions *
61	*********************************************************************************************************************************/
62	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
63	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
64	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
65	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
66	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
67	PPGMPTWALKGST pGstWalk);
68	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
69	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
70	PPGMPTWALKGST pGstWalkAll);
71	#endif
72	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
73	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
74	#ifdef PGM_WITH_PAGE_ZEROING_DETECTION
75	static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx);
76	#endif
77
78
79	/*
80	* Second level transation - EPT.
81	*/
82	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
83	# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
84	# include "PGMSlatDefs.h"
85	# include "PGMAllGstSlatEpt.cpp.h"
86	# undef PGM_SLAT_TYPE
87	#endif
88
89
90	/*
91	* Shadow - 32-bit mode
92	*/
93	#define PGM_SHW_TYPE PGM_TYPE_32BIT
94	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
95	#include "PGMAllShw.h"
96
97	/* Guest - real mode */
98	#define PGM_GST_TYPE PGM_TYPE_REAL
99	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
100	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
101	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
102	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
103	#include "PGMGstDefs.h"
104	#include "PGMAllGst.h"
105	#include "PGMAllBth.h"
106	#undef BTH_PGMPOOLKIND_PT_FOR_PT
107	#undef BTH_PGMPOOLKIND_ROOT
108	#undef PGM_BTH_NAME
109	#undef PGM_GST_TYPE
110	#undef PGM_GST_NAME
111
112	/* Guest - protected mode */
113	#define PGM_GST_TYPE PGM_TYPE_PROT
114	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
115	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
116	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
117	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
118	#include "PGMGstDefs.h"
119	#include "PGMAllGst.h"
120	#include "PGMAllBth.h"
121	#undef BTH_PGMPOOLKIND_PT_FOR_PT
122	#undef BTH_PGMPOOLKIND_ROOT
123	#undef PGM_BTH_NAME
124	#undef PGM_GST_TYPE
125	#undef PGM_GST_NAME
126
127	/* Guest - 32-bit mode */
128	#define PGM_GST_TYPE PGM_TYPE_32BIT
129	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
130	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
131	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
132	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
133	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
134	#include "PGMGstDefs.h"
135	#include "PGMAllGst.h"
136	#include "PGMAllBth.h"
137	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
138	#undef BTH_PGMPOOLKIND_PT_FOR_PT
139	#undef BTH_PGMPOOLKIND_ROOT
140	#undef PGM_BTH_NAME
141	#undef PGM_GST_TYPE
142	#undef PGM_GST_NAME
143
144	#undef PGM_SHW_TYPE
145	#undef PGM_SHW_NAME
146
147
148	/*
149	* Shadow - PAE mode
150	*/
151	#define PGM_SHW_TYPE PGM_TYPE_PAE
152	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
153	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
154	#include "PGMAllShw.h"
155
156	/* Guest - real mode */
157	#define PGM_GST_TYPE PGM_TYPE_REAL
158	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
159	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
160	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
161	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
162	#include "PGMGstDefs.h"
163	#include "PGMAllBth.h"
164	#undef BTH_PGMPOOLKIND_PT_FOR_PT
165	#undef BTH_PGMPOOLKIND_ROOT
166	#undef PGM_BTH_NAME
167	#undef PGM_GST_TYPE
168	#undef PGM_GST_NAME
169
170	/* Guest - protected mode */
171	#define PGM_GST_TYPE PGM_TYPE_PROT
172	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
173	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
174	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
175	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
176	#include "PGMGstDefs.h"
177	#include "PGMAllBth.h"
178	#undef BTH_PGMPOOLKIND_PT_FOR_PT
179	#undef BTH_PGMPOOLKIND_ROOT
180	#undef PGM_BTH_NAME
181	#undef PGM_GST_TYPE
182	#undef PGM_GST_NAME
183
184	/* Guest - 32-bit mode */
185	#define PGM_GST_TYPE PGM_TYPE_32BIT
186	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
187	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
188	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
189	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
190	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
191	#include "PGMGstDefs.h"
192	#include "PGMAllBth.h"
193	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
194	#undef BTH_PGMPOOLKIND_PT_FOR_PT
195	#undef BTH_PGMPOOLKIND_ROOT
196	#undef PGM_BTH_NAME
197	#undef PGM_GST_TYPE
198	#undef PGM_GST_NAME
199
200
201	/* Guest - PAE mode */
202	#define PGM_GST_TYPE PGM_TYPE_PAE
203	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
204	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
205	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
206	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
207	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
208	#include "PGMGstDefs.h"
209	#include "PGMAllGst.h"
210	#include "PGMAllBth.h"
211	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
212	#undef BTH_PGMPOOLKIND_PT_FOR_PT
213	#undef BTH_PGMPOOLKIND_ROOT
214	#undef PGM_BTH_NAME
215	#undef PGM_GST_TYPE
216	#undef PGM_GST_NAME
217
218	#undef PGM_SHW_TYPE
219	#undef PGM_SHW_NAME
220
221
222	/*
223	* Shadow - AMD64 mode
224	*/
225	#define PGM_SHW_TYPE PGM_TYPE_AMD64
226	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
227	#include "PGMAllShw.h"
228
229	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
230	/** @todo retire this hack. */
231	#define PGM_GST_TYPE PGM_TYPE_PROT
232	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
233	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
234	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
235	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
236	#include "PGMGstDefs.h"
237	#include "PGMAllBth.h"
238	#undef BTH_PGMPOOLKIND_PT_FOR_PT
239	#undef BTH_PGMPOOLKIND_ROOT
240	#undef PGM_BTH_NAME
241	#undef PGM_GST_TYPE
242	#undef PGM_GST_NAME
243
244	#ifdef VBOX_WITH_64_BITS_GUESTS
245	/* Guest - AMD64 mode */
246	# define PGM_GST_TYPE PGM_TYPE_AMD64
247	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
248	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
249	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
250	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
251	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
252	# include "PGMGstDefs.h"
253	# include "PGMAllGst.h"
254	# include "PGMAllBth.h"
255	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
256	# undef BTH_PGMPOOLKIND_PT_FOR_PT
257	# undef BTH_PGMPOOLKIND_ROOT
258	# undef PGM_BTH_NAME
259	# undef PGM_GST_TYPE
260	# undef PGM_GST_NAME
261	#endif /* VBOX_WITH_64_BITS_GUESTS */
262
263	#undef PGM_SHW_TYPE
264	#undef PGM_SHW_NAME
265
266
267	/*
268	* Shadow - 32-bit nested paging mode.
269	*/
270	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
271	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
272	#include "PGMAllShw.h"
273
274	/* Guest - real mode */
275	#define PGM_GST_TYPE PGM_TYPE_REAL
276	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
277	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
278	#include "PGMGstDefs.h"
279	#include "PGMAllBth.h"
280	#undef PGM_BTH_NAME
281	#undef PGM_GST_TYPE
282	#undef PGM_GST_NAME
283
284	/* Guest - protected mode */
285	#define PGM_GST_TYPE PGM_TYPE_PROT
286	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
287	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
288	#include "PGMGstDefs.h"
289	#include "PGMAllBth.h"
290	#undef PGM_BTH_NAME
291	#undef PGM_GST_TYPE
292	#undef PGM_GST_NAME
293
294	/* Guest - 32-bit mode */
295	#define PGM_GST_TYPE PGM_TYPE_32BIT
296	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
297	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
298	#include "PGMGstDefs.h"
299	#include "PGMAllBth.h"
300	#undef PGM_BTH_NAME
301	#undef PGM_GST_TYPE
302	#undef PGM_GST_NAME
303
304	/* Guest - PAE mode */
305	#define PGM_GST_TYPE PGM_TYPE_PAE
306	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
307	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
308	#include "PGMGstDefs.h"
309	#include "PGMAllBth.h"
310	#undef PGM_BTH_NAME
311	#undef PGM_GST_TYPE
312	#undef PGM_GST_NAME
313
314	#ifdef VBOX_WITH_64_BITS_GUESTS
315	/* Guest - AMD64 mode */
316	# define PGM_GST_TYPE PGM_TYPE_AMD64
317	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
318	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
319	# include "PGMGstDefs.h"
320	# include "PGMAllBth.h"
321	# undef PGM_BTH_NAME
322	# undef PGM_GST_TYPE
323	# undef PGM_GST_NAME
324	#endif /* VBOX_WITH_64_BITS_GUESTS */
325
326	#undef PGM_SHW_TYPE
327	#undef PGM_SHW_NAME
328
329
330	/*
331	* Shadow - PAE nested paging mode.
332	*/
333	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
334	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
335	#include "PGMAllShw.h"
336
337	/* Guest - real mode */
338	#define PGM_GST_TYPE PGM_TYPE_REAL
339	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
340	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
341	#include "PGMGstDefs.h"
342	#include "PGMAllBth.h"
343	#undef PGM_BTH_NAME
344	#undef PGM_GST_TYPE
345	#undef PGM_GST_NAME
346
347	/* Guest - protected mode */
348	#define PGM_GST_TYPE PGM_TYPE_PROT
349	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
350	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
351	#include "PGMGstDefs.h"
352	#include "PGMAllBth.h"
353	#undef PGM_BTH_NAME
354	#undef PGM_GST_TYPE
355	#undef PGM_GST_NAME
356
357	/* Guest - 32-bit mode */
358	#define PGM_GST_TYPE PGM_TYPE_32BIT
359	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
360	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
361	#include "PGMGstDefs.h"
362	#include "PGMAllBth.h"
363	#undef PGM_BTH_NAME
364	#undef PGM_GST_TYPE
365	#undef PGM_GST_NAME
366
367	/* Guest - PAE mode */
368	#define PGM_GST_TYPE PGM_TYPE_PAE
369	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
370	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
371	#include "PGMGstDefs.h"
372	#include "PGMAllBth.h"
373	#undef PGM_BTH_NAME
374	#undef PGM_GST_TYPE
375	#undef PGM_GST_NAME
376
377	#ifdef VBOX_WITH_64_BITS_GUESTS
378	/* Guest - AMD64 mode */
379	# define PGM_GST_TYPE PGM_TYPE_AMD64
380	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
381	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
382	# include "PGMGstDefs.h"
383	# include "PGMAllBth.h"
384	# undef PGM_BTH_NAME
385	# undef PGM_GST_TYPE
386	# undef PGM_GST_NAME
387	#endif /* VBOX_WITH_64_BITS_GUESTS */
388
389	#undef PGM_SHW_TYPE
390	#undef PGM_SHW_NAME
391
392
393	/*
394	* Shadow - AMD64 nested paging mode.
395	*/
396	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
397	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
398	#include "PGMAllShw.h"
399
400	/* Guest - real mode */
401	#define PGM_GST_TYPE PGM_TYPE_REAL
402	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
403	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
404	#include "PGMGstDefs.h"
405	#include "PGMAllBth.h"
406	#undef PGM_BTH_NAME
407	#undef PGM_GST_TYPE
408	#undef PGM_GST_NAME
409
410	/* Guest - protected mode */
411	#define PGM_GST_TYPE PGM_TYPE_PROT
412	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
413	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
414	#include "PGMGstDefs.h"
415	#include "PGMAllBth.h"
416	#undef PGM_BTH_NAME
417	#undef PGM_GST_TYPE
418	#undef PGM_GST_NAME
419
420	/* Guest - 32-bit mode */
421	#define PGM_GST_TYPE PGM_TYPE_32BIT
422	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
423	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
424	#include "PGMGstDefs.h"
425	#include "PGMAllBth.h"
426	#undef PGM_BTH_NAME
427	#undef PGM_GST_TYPE
428	#undef PGM_GST_NAME
429
430	/* Guest - PAE mode */
431	#define PGM_GST_TYPE PGM_TYPE_PAE
432	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
433	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
434	#include "PGMGstDefs.h"
435	#include "PGMAllBth.h"
436	#undef PGM_BTH_NAME
437	#undef PGM_GST_TYPE
438	#undef PGM_GST_NAME
439
440	#ifdef VBOX_WITH_64_BITS_GUESTS
441	/* Guest - AMD64 mode */
442	# define PGM_GST_TYPE PGM_TYPE_AMD64
443	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
444	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
445	# include "PGMGstDefs.h"
446	# include "PGMAllBth.h"
447	# undef PGM_BTH_NAME
448	# undef PGM_GST_TYPE
449	# undef PGM_GST_NAME
450	#endif /* VBOX_WITH_64_BITS_GUESTS */
451
452	#undef PGM_SHW_TYPE
453	#undef PGM_SHW_NAME
454
455
456	/*
457	* Shadow - EPT.
458	*/
459	#define PGM_SHW_TYPE PGM_TYPE_EPT
460	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
461	#include "PGMAllShw.h"
462
463	/* Guest - real mode */
464	#define PGM_GST_TYPE PGM_TYPE_REAL
465	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
466	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
467	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
468	#include "PGMGstDefs.h"
469	#include "PGMAllBth.h"
470	#undef BTH_PGMPOOLKIND_PT_FOR_PT
471	#undef PGM_BTH_NAME
472	#undef PGM_GST_TYPE
473	#undef PGM_GST_NAME
474
475	/* Guest - protected mode */
476	#define PGM_GST_TYPE PGM_TYPE_PROT
477	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
478	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
479	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
480	#include "PGMGstDefs.h"
481	#include "PGMAllBth.h"
482	#undef BTH_PGMPOOLKIND_PT_FOR_PT
483	#undef PGM_BTH_NAME
484	#undef PGM_GST_TYPE
485	#undef PGM_GST_NAME
486
487	/* Guest - 32-bit mode */
488	#define PGM_GST_TYPE PGM_TYPE_32BIT
489	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
490	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
491	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
492	#include "PGMGstDefs.h"
493	#include "PGMAllBth.h"
494	#undef BTH_PGMPOOLKIND_PT_FOR_PT
495	#undef PGM_BTH_NAME
496	#undef PGM_GST_TYPE
497	#undef PGM_GST_NAME
498
499	/* Guest - PAE mode */
500	#define PGM_GST_TYPE PGM_TYPE_PAE
501	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
502	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
503	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
504	#include "PGMGstDefs.h"
505	#include "PGMAllBth.h"
506	#undef BTH_PGMPOOLKIND_PT_FOR_PT
507	#undef PGM_BTH_NAME
508	#undef PGM_GST_TYPE
509	#undef PGM_GST_NAME
510
511	#ifdef VBOX_WITH_64_BITS_GUESTS
512	/* Guest - AMD64 mode */
513	# define PGM_GST_TYPE PGM_TYPE_AMD64
514	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
515	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
516	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
517	# include "PGMGstDefs.h"
518	# include "PGMAllBth.h"
519	# undef BTH_PGMPOOLKIND_PT_FOR_PT
520	# undef PGM_BTH_NAME
521	# undef PGM_GST_TYPE
522	# undef PGM_GST_NAME
523	#endif /* VBOX_WITH_64_BITS_GUESTS */
524
525	#undef PGM_SHW_TYPE
526	#undef PGM_SHW_NAME
527
528
529	/*
530	* Shadow - NEM / None.
531	*/
532	#define PGM_SHW_TYPE PGM_TYPE_NONE
533	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
534	#include "PGMAllShw.h"
535
536	/* Guest - real mode */
537	#define PGM_GST_TYPE PGM_TYPE_REAL
538	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
539	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
540	#include "PGMGstDefs.h"
541	#include "PGMAllBth.h"
542	#undef PGM_BTH_NAME
543	#undef PGM_GST_TYPE
544	#undef PGM_GST_NAME
545
546	/* Guest - protected mode */
547	#define PGM_GST_TYPE PGM_TYPE_PROT
548	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
549	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
550	#include "PGMGstDefs.h"
551	#include "PGMAllBth.h"
552	#undef PGM_BTH_NAME
553	#undef PGM_GST_TYPE
554	#undef PGM_GST_NAME
555
556	/* Guest - 32-bit mode */
557	#define PGM_GST_TYPE PGM_TYPE_32BIT
558	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
559	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
560	#include "PGMGstDefs.h"
561	#include "PGMAllBth.h"
562	#undef PGM_BTH_NAME
563	#undef PGM_GST_TYPE
564	#undef PGM_GST_NAME
565
566	/* Guest - PAE mode */
567	#define PGM_GST_TYPE PGM_TYPE_PAE
568	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
569	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
570	#include "PGMGstDefs.h"
571	#include "PGMAllBth.h"
572	#undef PGM_BTH_NAME
573	#undef PGM_GST_TYPE
574	#undef PGM_GST_NAME
575
576	#ifdef VBOX_WITH_64_BITS_GUESTS
577	/* Guest - AMD64 mode */
578	# define PGM_GST_TYPE PGM_TYPE_AMD64
579	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
580	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
581	# include "PGMGstDefs.h"
582	# include "PGMAllBth.h"
583	# undef PGM_BTH_NAME
584	# undef PGM_GST_TYPE
585	# undef PGM_GST_NAME
586	#endif /* VBOX_WITH_64_BITS_GUESTS */
587
588	#undef PGM_SHW_TYPE
589	#undef PGM_SHW_NAME
590
591
592
593	/**
594	* Guest mode data array.
595	*/
596	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
597	{
598	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
599	{
600	PGM_TYPE_REAL,
601	PGM_GST_NAME_REAL(GetPage),
602	PGM_GST_NAME_REAL(QueryPageFast),
603	PGM_GST_NAME_REAL(ModifyPage),
604	PGM_GST_NAME_REAL(Enter),
605	PGM_GST_NAME_REAL(Exit),
606	#ifdef IN_RING3
607	PGM_GST_NAME_REAL(Relocate),
608	#endif
609	},
610	{
611	PGM_TYPE_PROT,
612	PGM_GST_NAME_PROT(GetPage),
613	PGM_GST_NAME_PROT(QueryPageFast),
614	PGM_GST_NAME_PROT(ModifyPage),
615	PGM_GST_NAME_PROT(Enter),
616	PGM_GST_NAME_PROT(Exit),
617	#ifdef IN_RING3
618	PGM_GST_NAME_PROT(Relocate),
619	#endif
620	},
621	{
622	PGM_TYPE_32BIT,
623	PGM_GST_NAME_32BIT(GetPage),
624	PGM_GST_NAME_32BIT(QueryPageFast),
625	PGM_GST_NAME_32BIT(ModifyPage),
626	PGM_GST_NAME_32BIT(Enter),
627	PGM_GST_NAME_32BIT(Exit),
628	#ifdef IN_RING3
629	PGM_GST_NAME_32BIT(Relocate),
630	#endif
631	},
632	{
633	PGM_TYPE_PAE,
634	PGM_GST_NAME_PAE(GetPage),
635	PGM_GST_NAME_PAE(QueryPageFast),
636	PGM_GST_NAME_PAE(ModifyPage),
637	PGM_GST_NAME_PAE(Enter),
638	PGM_GST_NAME_PAE(Exit),
639	#ifdef IN_RING3
640	PGM_GST_NAME_PAE(Relocate),
641	#endif
642	},
643	#ifdef VBOX_WITH_64_BITS_GUESTS
644	{
645	PGM_TYPE_AMD64,
646	PGM_GST_NAME_AMD64(GetPage),
647	PGM_GST_NAME_AMD64(QueryPageFast),
648	PGM_GST_NAME_AMD64(ModifyPage),
649	PGM_GST_NAME_AMD64(Enter),
650	PGM_GST_NAME_AMD64(Exit),
651	# ifdef IN_RING3
652	PGM_GST_NAME_AMD64(Relocate),
653	# endif
654	},
655	#endif
656	};
657
658
659	/**
660	* The shadow mode data array.
661	*/
662	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
663	{
664	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
665	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
666	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
667	{
668	PGM_TYPE_32BIT,
669	PGM_SHW_NAME_32BIT(GetPage),
670	PGM_SHW_NAME_32BIT(ModifyPage),
671	PGM_SHW_NAME_32BIT(Enter),
672	PGM_SHW_NAME_32BIT(Exit),
673	#ifdef IN_RING3
674	PGM_SHW_NAME_32BIT(Relocate),
675	#endif
676	},
677	{
678	PGM_TYPE_PAE,
679	PGM_SHW_NAME_PAE(GetPage),
680	PGM_SHW_NAME_PAE(ModifyPage),
681	PGM_SHW_NAME_PAE(Enter),
682	PGM_SHW_NAME_PAE(Exit),
683	#ifdef IN_RING3
684	PGM_SHW_NAME_PAE(Relocate),
685	#endif
686	},
687	{
688	PGM_TYPE_AMD64,
689	PGM_SHW_NAME_AMD64(GetPage),
690	PGM_SHW_NAME_AMD64(ModifyPage),
691	PGM_SHW_NAME_AMD64(Enter),
692	PGM_SHW_NAME_AMD64(Exit),
693	#ifdef IN_RING3
694	PGM_SHW_NAME_AMD64(Relocate),
695	#endif
696	},
697	{
698	PGM_TYPE_NESTED_32BIT,
699	PGM_SHW_NAME_NESTED_32BIT(GetPage),
700	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
701	PGM_SHW_NAME_NESTED_32BIT(Enter),
702	PGM_SHW_NAME_NESTED_32BIT(Exit),
703	#ifdef IN_RING3
704	PGM_SHW_NAME_NESTED_32BIT(Relocate),
705	#endif
706	},
707	{
708	PGM_TYPE_NESTED_PAE,
709	PGM_SHW_NAME_NESTED_PAE(GetPage),
710	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
711	PGM_SHW_NAME_NESTED_PAE(Enter),
712	PGM_SHW_NAME_NESTED_PAE(Exit),
713	#ifdef IN_RING3
714	PGM_SHW_NAME_NESTED_PAE(Relocate),
715	#endif
716	},
717	{
718	PGM_TYPE_NESTED_AMD64,
719	PGM_SHW_NAME_NESTED_AMD64(GetPage),
720	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
721	PGM_SHW_NAME_NESTED_AMD64(Enter),
722	PGM_SHW_NAME_NESTED_AMD64(Exit),
723	#ifdef IN_RING3
724	PGM_SHW_NAME_NESTED_AMD64(Relocate),
725	#endif
726	},
727	{
728	PGM_TYPE_EPT,
729	PGM_SHW_NAME_EPT(GetPage),
730	PGM_SHW_NAME_EPT(ModifyPage),
731	PGM_SHW_NAME_EPT(Enter),
732	PGM_SHW_NAME_EPT(Exit),
733	#ifdef IN_RING3
734	PGM_SHW_NAME_EPT(Relocate),
735	#endif
736	},
737	{
738	PGM_TYPE_NONE,
739	PGM_SHW_NAME_NONE(GetPage),
740	PGM_SHW_NAME_NONE(ModifyPage),
741	PGM_SHW_NAME_NONE(Enter),
742	PGM_SHW_NAME_NONE(Exit),
743	#ifdef IN_RING3
744	PGM_SHW_NAME_NONE(Relocate),
745	#endif
746	},
747	};
748
749
750	/**
751	* The guest+shadow mode data array.
752	*/
753	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
754	{
755	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
756	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
757	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
758	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
759
760	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
761	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
762	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
763	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
764
765	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
766	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
767	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
768	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
769
770	#elif defined(IN_RING3) && defined(VBOX_STRICT)
771	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
772	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
773	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
774
775	#else
776	# error "Misconfig."
777	#endif
778
779	/* 32-bit shadow paging mode: */
780	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
781	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
782	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
783	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
784	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
785	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
786	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
787	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
788	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
789	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
790	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
791
792	/* PAE shadow paging mode: */
793	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
794	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
795	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
796	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
797	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
798	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
799	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
800	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
801	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
802	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
803	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
804
805	/* AMD64 shadow paging mode: */
806	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
807	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
808	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
809	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
810	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
811	#ifdef VBOX_WITH_64_BITS_GUESTS
812	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
813	#else
814	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
815	#endif
816	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
817	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
818	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
819	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
820	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
821
822	/* 32-bit nested paging mode: */
823	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
824	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
825	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
826	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
827	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
828	#ifdef VBOX_WITH_64_BITS_GUESTS
829	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
830	#else
831	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
832	#endif
833	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
834	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
835	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
836	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
837	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
838
839	/* PAE nested paging mode: */
840	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
841	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
842	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
843	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
844	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
845	#ifdef VBOX_WITH_64_BITS_GUESTS
846	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
847	#else
848	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
849	#endif
850	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
851	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
852	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
853	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
854	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
855
856	/* AMD64 nested paging mode: */
857	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
858	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
859	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
860	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
861	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
862	#ifdef VBOX_WITH_64_BITS_GUESTS
863	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
864	#else
865	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
866	#endif
867	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
868	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
869	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
870	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
871	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
872
873	/* EPT nested paging mode: */
874	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
875	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
876	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
877	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
878	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
879	#ifdef VBOX_WITH_64_BITS_GUESTS
880	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
881	#else
882	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
883	#endif
884	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
885	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
886	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
887	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
888	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
889
890	/* NONE / NEM: */
891	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
892	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
893	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
894	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
895	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
896	#ifdef VBOX_WITH_64_BITS_GUESTS
897	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
898	#else
899	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
900	#endif
901	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
902	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
903	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
904	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
905	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
906
907
908	#undef PGMMODEDATABTH_ENTRY
909	#undef PGMMODEDATABTH_NULL_ENTRY
910	};
911
912
913	/** Mask array used by pgmGetCr3MaskForMode.
914	* X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
915	static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
916	{
917	/* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK,
918	/* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK,
919	/* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK,
920	/* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK,
921	/* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK,
922	/* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK,
923	/* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
924	/* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK,
925	/* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK,
926	/* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK,
927	/* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
928	/* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK,
929	/* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK,
930	};
931
932
933	/**
934	* Gets the physical address mask for CR3 in the given paging mode.
935	*
936	* The mask is for eliminating flags and other stuff in CR3/EPTP when
937	* extracting the physical address. It is not for validating whether there are
938	* reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed
939	* it to PGM checked for reserved bits, including reserved physical address
940	* bits.
941	*
942	* @returns The CR3 mask.
943	* @param enmMode The paging mode.
944	* @param enmSlatMode The second-level address translation mode.
945	*/
946	DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
947	{
948	if (enmSlatMode == PGMSLAT_DIRECT)
949	{
950	Assert(enmMode != PGMMODE_EPT);
951	return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
952	}
953	Assert(enmSlatMode == PGMSLAT_EPT);
954	return X86_CR3_EPT_PAGE_MASK;
955	}
956
957
958	/**
959	* Gets the masked CR3 value according to the current guest paging mode.
960	*
961	* See disclaimer in pgmGetCr3MaskForMode.
962	*
963	* @returns The masked PGM CR3 value.
964	* @param pVCpu The cross context virtual CPU structure.
965	* @param uCr3 The raw guest CR3 value.
966	*/
967	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
968	{
969	uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
970	RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
971	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
972	return GCPhysCR3;
973	}
974
975
976	#ifdef IN_RING0
977	/**
978	* #PF Handler.
979	*
980	* @returns VBox status code (appropriate for trap handling and GC return).
981	* @param pVCpu The cross context virtual CPU structure.
982	* @param uErr The trap error code.
983	* @param pCtx Pointer to the register context for the CPU.
984	* @param pvFault The fault address.
985	*/
986	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
987	{
988	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
989
990	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
991	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
992	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
993
994
995	# ifdef VBOX_WITH_STATISTICS
996	/*
997	* Error code stats.
998	*/
999	if (uErr & X86_TRAP_PF_US)
1000	{
1001	if (!(uErr & X86_TRAP_PF_P))
1002	{
1003	if (uErr & X86_TRAP_PF_RW)
1004	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
1005	else
1006	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
1007	}
1008	else if (uErr & X86_TRAP_PF_RW)
1009	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1010	else if (uErr & X86_TRAP_PF_RSVD)
1011	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1012	else if (uErr & X86_TRAP_PF_ID)
1013	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1014	else
1015	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1016	}
1017	else
1018	{ /* Supervisor */
1019	if (!(uErr & X86_TRAP_PF_P))
1020	{
1021	if (uErr & X86_TRAP_PF_RW)
1022	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1023	else
1024	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1025	}
1026	else if (uErr & X86_TRAP_PF_RW)
1027	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1028	else if (uErr & X86_TRAP_PF_ID)
1029	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1030	else if (uErr & X86_TRAP_PF_RSVD)
1031	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1032	}
1033	# endif /* VBOX_WITH_STATISTICS */
1034
1035	/*
1036	* Call the worker.
1037	*/
1038	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1039	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1040	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
1041	bool fLockTaken = false;
1042	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
1043	if (fLockTaken)
1044	{
1045	PGM_LOCK_ASSERT_OWNER(pVM);
1046	PGM_UNLOCK(pVM);
1047	}
1048	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
1049
1050	/*
1051	* Return code tweaks.
1052	*/
1053	if (rc != VINF_SUCCESS)
1054	{
1055	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1056	rc = VINF_SUCCESS;
1057
1058	/* Note: hack alert for difficult to reproduce problem. */
1059	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1060	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1061	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1062	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1063	{
1064	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
1065	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
1066	rc = VINF_SUCCESS;
1067	}
1068	}
1069
1070	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
1071	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1072	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1073	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1074	return rc;
1075	}
1076	#endif /* IN_RING0 */
1077
1078
1079	/**
1080	* Prefetch a page
1081	*
1082	* Typically used to sync commonly used pages before entering raw mode
1083	* after a CR3 reload.
1084	*
1085	* @returns VBox status code suitable for scheduling.
1086	* @retval VINF_SUCCESS on success.
1087	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1088	* @param pVCpu The cross context virtual CPU structure.
1089	* @param GCPtrPage Page to invalidate.
1090	*/
1091	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1092	{
1093	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1094
1095	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1096	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1097	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1098	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1099
1100	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1101	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1102	return rc;
1103	}
1104
1105
1106	/**
1107	* Emulation of the invlpg instruction (HC only actually).
1108	*
1109	* @returns Strict VBox status code, special care required.
1110	* @retval VINF_PGM_SYNC_CR3 - handled.
1111	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1112	*
1113	* @param pVCpu The cross context virtual CPU structure.
1114	* @param GCPtrPage Page to invalidate.
1115	*
1116	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1117	* safe, but there could be edge cases!
1118	*
1119	* @todo Flush page or page directory only if necessary!
1120	* @todo VBOXSTRICTRC
1121	*/
1122	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1123	{
1124	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1125	int rc;
1126	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1127
1128	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1129
1130	/*
1131	* Call paging mode specific worker.
1132	*/
1133	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1134	PGM_LOCK_VOID(pVM);
1135
1136	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1137	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1138	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1139	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1140
1141	PGM_UNLOCK(pVM);
1142	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1143
1144	/* Ignore all irrelevant error codes. */
1145	if ( rc == VERR_PAGE_NOT_PRESENT
1146	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1147	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1148	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1149	rc = VINF_SUCCESS;
1150
1151	return rc;
1152	}
1153
1154
1155	/**
1156	* Executes an instruction using the interpreter.
1157	*
1158	* @returns VBox status code (appropriate for trap handling and GC return).
1159	* @param pVCpu The cross context virtual CPU structure.
1160	* @param pvFault Fault address.
1161	*/
1162	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
1163	{
1164	RT_NOREF(pvFault);
1165	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
1166	if (rc == VERR_EM_INTERPRETER)
1167	rc = VINF_EM_RAW_EMULATE_INSTR;
1168	if (rc != VINF_SUCCESS)
1169	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1170	return rc;
1171	}
1172
1173
1174	/**
1175	* Gets effective page information (from the VMM page directory).
1176	*
1177	* @returns VBox status code.
1178	* @param pVCpu The cross context virtual CPU structure.
1179	* @param GCPtr Guest Context virtual address of the page.
1180	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1181	* @param pHCPhys Where to store the HC physical address of the page.
1182	* This is page aligned.
1183	* @remark You should use PGMMapGetPage() for pages in a mapping.
1184	*/
1185	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1186	{
1187	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1188	PGM_LOCK_VOID(pVM);
1189
1190	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1191	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1192	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1193	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1194
1195	PGM_UNLOCK(pVM);
1196	return rc;
1197	}
1198
1199
1200	/**
1201	* Modify page flags for a range of pages in the shadow context.
1202	*
1203	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1204	*
1205	* @returns VBox status code.
1206	* @param pVCpu The cross context virtual CPU structure.
1207	* @param GCPtr Virtual address of the first page in the range.
1208	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1209	* @param fMask The AND mask - page flags X86_PTE_*.
1210	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1211	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1212	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1213	*/
1214	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1215	{
1216	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1217	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1218
1219	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1220
1221	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1222	PGM_LOCK_VOID(pVM);
1223
1224	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1225	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1226	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1227	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1228
1229	PGM_UNLOCK(pVM);
1230	return rc;
1231	}
1232
1233
1234	/**
1235	* Changing the page flags for a single page in the shadow page tables so as to
1236	* make it read-only.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu The cross context virtual CPU structure.
1240	* @param GCPtr Virtual address of the first page in the range.
1241	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1242	*/
1243	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1244	{
1245	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1246	}
1247
1248
1249	/**
1250	* Changing the page flags for a single page in the shadow page tables so as to
1251	* make it writable.
1252	*
1253	* The call must know with 101% certainty that the guest page tables maps this
1254	* as writable too. This function will deal shared, zero and write monitored
1255	* pages.
1256	*
1257	* @returns VBox status code.
1258	* @param pVCpu The cross context virtual CPU structure.
1259	* @param GCPtr Virtual address of the first page in the range.
1260	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1261	*/
1262	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1263	{
1264	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1265	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1266	return VINF_SUCCESS;
1267	}
1268
1269
1270	/**
1271	* Changing the page flags for a single page in the shadow page tables so as to
1272	* make it not present.
1273	*
1274	* @returns VBox status code.
1275	* @param pVCpu The cross context virtual CPU structure.
1276	* @param GCPtr Virtual address of the first page in the range.
1277	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1278	*/
1279	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1280	{
1281	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1282	}
1283
1284	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
1285
1286	/**
1287	* Changing the page flags for a single page in the shadow page tables so as to
1288	* make it supervisor and writable.
1289	*
1290	* This if for dealing with CR0.WP=0 and readonly user pages.
1291	*
1292	* @returns VBox status code.
1293	* @param pVCpu The cross context virtual CPU structure.
1294	* @param GCPtr Virtual address of the first page in the range.
1295	* @param fBigPage Whether or not this is a big page. If it is, we have to
1296	* change the shadow PDE as well. If it isn't, the caller
1297	* has checked that the shadow PDE doesn't need changing.
1298	* We ASSUME 4KB pages backing the big page here!
1299	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1300	*/
1301	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1302	{
1303	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1304	if (rc == VINF_SUCCESS && fBigPage)
1305	{
1306	/* this is a bit ugly... */
1307	switch (pVCpu->pgm.s.enmShadowMode)
1308	{
1309	case PGMMODE_32_BIT:
1310	{
1311	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1312	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1313	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1314	pPde->u \|= X86_PDE_RW;
1315	Log(("-> PDE=%#llx (32)\n", pPde->u));
1316	break;
1317	}
1318	case PGMMODE_PAE:
1319	case PGMMODE_PAE_NX:
1320	{
1321	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1322	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1323	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1324	pPde->u \|= X86_PDE_RW;
1325	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1326	break;
1327	}
1328	default:
1329	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1330	}
1331	}
1332	return rc;
1333	}
1334
1335
1336	/**
1337	* Gets the shadow page directory for the specified address, PAE.
1338	*
1339	* @returns Pointer to the shadow PD.
1340	* @param pVCpu The cross context virtual CPU structure.
1341	* @param GCPtr The address.
1342	* @param uGstPdpe Guest PDPT entry. Valid.
1343	* @param ppPD Receives address of page directory
1344	*/
1345	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1346	{
1347	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1348	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1349	PPGMPOOLPAGE pShwPage;
1350	int rc;
1351	PGM_LOCK_ASSERT_OWNER(pVM);
1352
1353
1354	/* Allocate page directory if not present. */
1355	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1356	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1357	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1358	X86PGPAEUINT const uPdpe = pPdpe->u;
1359	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1360	{
1361	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1362	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1363	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1364
1365	pgmPoolCacheUsed(pPool, pShwPage);
1366
1367	/* Update the entry if necessary. */
1368	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1369	if (uPdpeNew == uPdpe)
1370	{ /* likely */ }
1371	else
1372	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1373	}
1374	else
1375	{
1376	RTGCPTR64 GCPdPt;
1377	PGMPOOLKIND enmKind;
1378	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1379	{
1380	/* AMD-V nested paging or real/protected mode without paging. */
1381	GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1382	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1383	}
1384	else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1385	{
1386	if (uGstPdpe & X86_PDPE_P)
1387	{
1388	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1389	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1390	}
1391	else
1392	{
1393	/* PD not present; guest must reload CR3 to change it.
1394	* No need to monitor anything in this case. */
1395	/** @todo r=bird: WTF is hit?!? */
1396	/Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? /
1397	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1398	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1399	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1400	}
1401	}
1402	else
1403	{
1404	GCPdPt = CPUMGetGuestCR3(pVCpu);
1405	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1406	}
1407
1408	/* Create a reference back to the PDPT by using the index in its shadow page. */
1409	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1410	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1411	&pShwPage);
1412	AssertRCReturn(rc, rc);
1413
1414	/* Hook it up. */
1415	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1416	}
1417	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1418
1419	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1420	return VINF_SUCCESS;
1421	}
1422
1423
1424	/**
1425	* Gets the pointer to the shadow page directory entry for an address, PAE.
1426	*
1427	* @returns Pointer to the PDE.
1428	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1429	* @param GCPtr The address.
1430	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1431	*/
1432	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1433	{
1434	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1435	PGM_LOCK_ASSERT_OWNER(pVM);
1436
1437	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1438	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1439	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1440	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1441	if (!(uPdpe & X86_PDPE_P))
1442	{
1443	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1444	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1445	}
1446	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1447
1448	/* Fetch the pgm pool shadow descriptor. */
1449	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1450	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1451
1452	*ppShwPde = pShwPde;
1453	return VINF_SUCCESS;
1454	}
1455
1456
1457	/**
1458	* Syncs the SHADOW page directory pointer for the specified address.
1459	*
1460	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1461	*
1462	* The caller is responsible for making sure the guest has a valid PD before
1463	* calling this function.
1464	*
1465	* @returns VBox status code.
1466	* @param pVCpu The cross context virtual CPU structure.
1467	* @param GCPtr The address.
1468	* @param uGstPml4e Guest PML4 entry (valid).
1469	* @param uGstPdpe Guest PDPT entry (valid).
1470	* @param ppPD Receives address of page directory
1471	*/
1472	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1473	{
1474	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1475	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1476	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1477	int rc;
1478
1479	PGM_LOCK_ASSERT_OWNER(pVM);
1480
1481	/*
1482	* PML4.
1483	*/
1484	PPGMPOOLPAGE pShwPage;
1485	{
1486	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1487	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1488	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1489	X86PGPAEUINT const uPml4e = pPml4e->u;
1490
1491	/* Allocate page directory pointer table if not present. */
1492	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1493	{
1494	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1495	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1496
1497	pgmPoolCacheUsed(pPool, pShwPage);
1498
1499	/* Update the entry if needed. */
1500	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1501	\| (uPml4e & PGM_PML4_FLAGS);
1502	if (uPml4e == uPml4eNew)
1503	{ /* likely */ }
1504	else
1505	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1506	}
1507	else
1508	{
1509	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1510
1511	RTGCPTR64 GCPml4;
1512	PGMPOOLKIND enmKind;
1513	if (fNestedPagingOrNoGstPaging)
1514	{
1515	/* AMD-V nested paging or real/protected mode without paging */
1516	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1517	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1518	}
1519	else
1520	{
1521	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1522	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1523	}
1524
1525	/* Create a reference back to the PDPT by using the index in its shadow page. */
1526	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1527	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1528	&pShwPage);
1529	AssertRCReturn(rc, rc);
1530
1531	/* Hook it up. */
1532	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1533	\| (uPml4e & PGM_PML4_FLAGS));
1534	}
1535	}
1536
1537	/*
1538	* PDPT.
1539	*/
1540	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1541	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1542	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1543	X86PGPAEUINT const uPdpe = pPdpe->u;
1544
1545	/* Allocate page directory if not present. */
1546	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1547	{
1548	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1549	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1550
1551	pgmPoolCacheUsed(pPool, pShwPage);
1552
1553	/* Update the entry if needed. */
1554	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1555	\| (uPdpe & PGM_PDPT_FLAGS);
1556	if (uPdpe == uPdpeNew)
1557	{ /* likely */ }
1558	else
1559	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1560	}
1561	else
1562	{
1563	RTGCPTR64 GCPdPt;
1564	PGMPOOLKIND enmKind;
1565	if (fNestedPagingOrNoGstPaging)
1566	{
1567	/* AMD-V nested paging or real/protected mode without paging */
1568	GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1569	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1570	}
1571	else
1572	{
1573	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1574	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1575	}
1576
1577	/* Create a reference back to the PDPT by using the index in its shadow page. */
1578	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1579	pShwPage->idx, iPdPt, false /fLockPage/,
1580	&pShwPage);
1581	AssertRCReturn(rc, rc);
1582
1583	/* Hook it up. */
1584	ASMAtomicWriteU64(&pPdpe->u,
1585	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1586	}
1587
1588	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1589	return VINF_SUCCESS;
1590	}
1591
1592
1593	/**
1594	* Gets the SHADOW page directory pointer for the specified address (long mode).
1595	*
1596	* @returns VBox status code.
1597	* @param pVCpu The cross context virtual CPU structure.
1598	* @param GCPtr The address.
1599	* @param ppPml4e Receives the address of the page map level 4 entry.
1600	* @param ppPdpt Receives the address of the page directory pointer table.
1601	* @param ppPD Receives the address of the page directory.
1602	*/
1603	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1604	{
1605	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1606	PGM_LOCK_ASSERT_OWNER(pVM);
1607
1608	/*
1609	* PML4
1610	*/
1611	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1612	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1613	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1614	if (ppPml4e)
1615	*ppPml4e = (PX86PML4E)pPml4e;
1616	X86PGPAEUINT const uPml4e = pPml4e->u;
1617	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1618	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1619	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1620
1621	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1622	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1623	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1624
1625	/*
1626	* PDPT
1627	*/
1628	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1629	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1630	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1631	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1632	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1633
1634	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1635	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1636
1637	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1638	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1639	return VINF_SUCCESS;
1640	}
1641
1642
1643	/**
1644	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1645	* backing pages in case the PDPT or PML4 entry is missing.
1646	*
1647	* @returns VBox status code.
1648	* @param pVCpu The cross context virtual CPU structure.
1649	* @param GCPtr The address.
1650	* @param ppPdpt Receives address of pdpt
1651	* @param ppPD Receives address of page directory
1652	*/
1653	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1654	{
1655	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1656	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1657	int rc;
1658
1659	Assert(pVM->pgm.s.fNestedPaging);
1660	PGM_LOCK_ASSERT_OWNER(pVM);
1661
1662	/*
1663	* PML4 level.
1664	*/
1665	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1666	Assert(pPml4);
1667
1668	/* Allocate page directory pointer table if not present. */
1669	PPGMPOOLPAGE pShwPage;
1670	{
1671	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1672	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1673	EPTPML4E Pml4e;
1674	Pml4e.u = pPml4e->u;
1675	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1676	{
1677	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1678	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1679	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1680	&pShwPage);
1681	AssertRCReturn(rc, rc);
1682
1683	/* Hook up the new PDPT now. */
1684	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1685	}
1686	else
1687	{
1688	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1689	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1690
1691	pgmPoolCacheUsed(pPool, pShwPage);
1692
1693	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1694	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1695	{ }
1696	else
1697	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1698	}
1699	}
1700
1701	/*
1702	* PDPT level.
1703	*/
1704	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1705	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1706	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1707
1708	if (ppPdpt)
1709	*ppPdpt = pPdpt;
1710
1711	/* Allocate page directory if not present. */
1712	EPTPDPTE Pdpe;
1713	Pdpe.u = pPdpe->u;
1714	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1715	{
1716	RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1717	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1718	pShwPage->idx, iPdPt, false /fLockPage/,
1719	&pShwPage);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Hook up the new PD now. */
1723	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1724	}
1725	else
1726	{
1727	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1728	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1729
1730	pgmPoolCacheUsed(pPool, pShwPage);
1731
1732	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1733	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1734	{ }
1735	else
1736	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1737	}
1738
1739	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1740	return VINF_SUCCESS;
1741	}
1742
1743
1744	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1745	/**
1746	* Syncs the SHADOW nested-guest page directory pointer for the specified address.
1747	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1748	*
1749	* @returns VBox status code.
1750	* @param pVCpu The cross context virtual CPU structure.
1751	* @param GCPhysNested The nested-guest physical address.
1752	* @param ppPdpt Where to store the PDPT. Optional, can be NULL.
1753	* @param ppPD Where to store the PD. Optional, can be NULL.
1754	* @param pGstWalkAll The guest walk info.
1755	*/
1756	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
1757	PPGMPTWALKGST pGstWalkAll)
1758	{
1759	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1760	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1761	int rc;
1762
1763	PPGMPOOLPAGE pShwPage;
1764	Assert(pVM->pgm.s.fNestedPaging);
1765	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
1766	PGM_LOCK_ASSERT_OWNER(pVM);
1767
1768	/*
1769	* PML4 level.
1770	*/
1771	{
1772	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1773	Assert(pPml4);
1774
1775	/* Allocate page directory pointer table if not present. */
1776	{
1777	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
1778	const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1779	PEPTPML4E pPml4e = &pPml4->a[iPml4e];
1780
1781	if (!(pPml4e->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1782	{
1783	RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
1784	rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
1785	PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /fLockPage/,
1786	&pShwPage);
1787	AssertRCReturn(rc, rc);
1788
1789	/* Hook up the new PDPT now. */
1790	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1791	}
1792	else
1793	{
1794	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1795	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1796
1797	pgmPoolCacheUsed(pPool, pShwPage);
1798
1799	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1800	if (pPml4e->u != (pShwPage->Core.Key \| fShwFlags))
1801	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1802	}
1803	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1804	Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
1805	}
1806	}
1807
1808	/*
1809	* PDPT level.
1810	*/
1811	{
1812	AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
1813
1814	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1815	if (ppPdpt)
1816	*ppPdpt = pPdpt;
1817
1818	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
1819	const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1820	PEPTPDPTE pPdpte = &pPdpt->a[iPdPte];
1821
1822	if (!(pPdpte->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1823	{
1824	RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
1825	rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1826	pShwPage->idx, iPdPte, false /fLockPage/, &pShwPage);
1827	AssertRCReturn(rc, rc);
1828
1829	/* Hook up the new PD now. */
1830	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1831	}
1832	else
1833	{
1834	pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
1835	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1836
1837	pgmPoolCacheUsed(pPool, pShwPage);
1838
1839	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1840	if (pPdpte->u != (pShwPage->Core.Key \| fShwFlags))
1841	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1842	}
1843	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1844	Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
1845
1846	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1847	}
1848
1849	return VINF_SUCCESS;
1850	}
1851	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1852
1853
1854	# ifdef IN_RING0
1855	/**
1856	* Synchronizes a range of nested page table entries.
1857	*
1858	* The caller must own the PGM lock.
1859	*
1860	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1861	* @param GCPhys Where to start.
1862	* @param cPages How many pages which entries should be synced.
1863	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1864	* host paging mode for AMD-V).
1865	*/
1866	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1867	{
1868	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1869
1870	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1871	int rc;
1872	switch (enmShwPagingMode)
1873	{
1874	case PGMMODE_32_BIT:
1875	{
1876	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1877	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1878	break;
1879	}
1880
1881	case PGMMODE_PAE:
1882	case PGMMODE_PAE_NX:
1883	{
1884	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1885	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1886	break;
1887	}
1888
1889	case PGMMODE_AMD64:
1890	case PGMMODE_AMD64_NX:
1891	{
1892	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1893	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1894	break;
1895	}
1896
1897	case PGMMODE_EPT:
1898	{
1899	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1900	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1901	break;
1902	}
1903
1904	default:
1905	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1906	}
1907	return rc;
1908	}
1909	# endif /* IN_RING0 */
1910
1911	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
1912
1913	/**
1914	* Gets effective Guest OS page information.
1915	*
1916	* @returns VBox status code.
1917	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1918	* @param GCPtr Guest Context virtual address of the page.
1919	* @param pWalk Where to store the page walk information.
1920	* @thread EMT(pVCpu)
1921	*/
1922	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
1923	{
1924	VMCPU_ASSERT_EMT(pVCpu);
1925	Assert(pWalk);
1926	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1927	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1928	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1929	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
1930	}
1931
1932
1933	/**
1934	* Gets effective Guest OS page information.
1935	*
1936	* @returns VBox status code.
1937	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1938	* @param GCPtr Guest Context virtual address of the page.
1939	* @param fFlags PGMQPAGE_F_XXX. If zero, no accessed or dirty bits will
1940	* be set.
1941	* @param pWalk Where to store the page walk information.
1942	* @thread EMT(pVCpu)
1943	*/
1944	VMM_INT_DECL(int) PGMGstQueryPageFast(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
1945	{
1946	VMCPU_ASSERT_EMT(pVCpu);
1947	Assert(pWalk);
1948	Assert(!(fFlags & ~(PGMQPAGE_F_VALID_MASK)));
1949	Assert(!(fFlags & PGMQPAGE_F_EXECUTE) \|\| !(fFlags & PGMQPAGE_F_WRITE));
1950	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1951	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1952	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1953	return g_aPgmGuestModeData[idx].pfnQueryPageFast(pVCpu, GCPtr, fFlags, pWalk);
1954	}
1955
1956
1957	/**
1958	* Maps the guest CR3.
1959	*
1960	* @returns VBox status code.
1961	* @param pVCpu The cross context virtual CPU structure.
1962	* @param GCPhysCr3 The guest CR3 value.
1963	* @param pHCPtrGuestCr3 Where to store the mapped memory.
1964	*/
1965	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
1966	{
1967	/** @todo this needs some reworking wrt. locking? */
1968	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1969	PGM_LOCK_VOID(pVM);
1970	PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
1971	AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
1972
1973	RTHCPTR HCPtrGuestCr3;
1974	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
1975	PGM_UNLOCK(pVM);
1976
1977	*pHCPtrGuestCr3 = HCPtrGuestCr3;
1978	return rc;
1979	}
1980
1981
1982	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1983	/**
1984	* Unmaps the guest CR3.
1985	*
1986	* @returns VBox status code.
1987	* @param pVCpu The cross context virtual CPU structure.
1988	*/
1989	DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
1990	{
1991	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1992	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1993	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
1994	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
1995	}
1996	#endif
1997
1998
1999	/**
2000	* Performs a guest page table walk.
2001	*
2002	* The guest should be in paged protect mode or long mode when making a call to
2003	* this function.
2004	*
2005	* @returns VBox status code.
2006	* @retval VINF_SUCCESS on success.
2007	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2008	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2009	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2010	*
2011	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2012	* @param GCPtr The guest virtual address to walk by.
2013	* @param pWalk Where to return the walk result. This is valid for some
2014	* error codes as well.
2015	* @param pGstWalk The guest mode specific page walk information.
2016	*/
2017	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2018	{
2019	VMCPU_ASSERT_EMT(pVCpu);
2020	switch (pVCpu->pgm.s.enmGuestMode)
2021	{
2022	case PGMMODE_32_BIT:
2023	pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
2024	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
2025
2026	case PGMMODE_PAE:
2027	case PGMMODE_PAE_NX:
2028	pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
2029	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
2030
2031	case PGMMODE_AMD64:
2032	case PGMMODE_AMD64_NX:
2033	pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
2034	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
2035
2036	case PGMMODE_REAL:
2037	case PGMMODE_PROTECTED:
2038	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2039	return VERR_PGM_NOT_USED_IN_MODE;
2040
2041	case PGMMODE_EPT:
2042	case PGMMODE_NESTED_32BIT:
2043	case PGMMODE_NESTED_PAE:
2044	case PGMMODE_NESTED_AMD64:
2045	default:
2046	AssertFailed();
2047	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2048	return VERR_PGM_NOT_USED_IN_MODE;
2049	}
2050	}
2051
2052
2053	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2054	/**
2055	* Performs a guest second-level address translation (SLAT).
2056	*
2057	* @returns VBox status code.
2058	* @retval VINF_SUCCESS on success.
2059	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2060	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2061	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2062	*
2063	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2064	* @param GCPhysNested The nested-guest physical address being translated.
2065	* @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the
2066	* cause for this translation.
2067	* @param GCPtrNested The nested-guest virtual address that initiated the
2068	* SLAT. If none, pass 0 (and not NIL_RTGCPTR).
2069	* @param pWalk Where to return the walk result. This is updated for
2070	* all error codes other than
2071	* VERR_PGM_NOT_USED_IN_MODE.
2072	* @param pGstWalk Where to store the second-level paging-mode specific
2073	* walk info.
2074	*/
2075	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
2076	PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2077	{
2078	/* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
2079	Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
2080	&& pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
2081	AssertPtr(pWalk);
2082	AssertPtr(pGstWalk);
2083	switch (pVCpu->pgm.s.enmGuestSlatMode)
2084	{
2085	case PGMSLAT_EPT:
2086	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
2087	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
2088
2089	default:
2090	AssertFailed();
2091	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2092	return VERR_PGM_NOT_USED_IN_MODE;
2093	}
2094	}
2095	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
2096
2097
2098	/**
2099	* Tries to continue the previous walk.
2100	*
2101	* @note Requires the caller to hold the PGM lock from the first
2102	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
2103	* we cannot use the pointers.
2104	*
2105	* @returns VBox status code.
2106	* @retval VINF_SUCCESS on success.
2107	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2108	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2109	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2110	*
2111	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2112	* @param GCPtr The guest virtual address to walk by.
2113	* @param pWalk Pointer to the previous walk result and where to return
2114	* the result of this walk. This is valid for some error
2115	* codes as well.
2116	* @param pGstWalk The guest-mode specific walk information.
2117	*/
2118	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2119	{
2120	/*
2121	* We can only handle successfully walks.
2122	* We also limit ourselves to the next page.
2123	*/
2124	if ( pWalk->fSucceeded
2125	&& GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
2126	{
2127	Assert(pWalk->uLevel == 0);
2128	if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2129	{
2130	/*
2131	* AMD64
2132	*/
2133	if (!pWalk->fGigantPage && !pWalk->fBigPage)
2134	{
2135	/*
2136	* We fall back to full walk if the PDE table changes, if any
2137	* reserved bits are set, or if the effective page access changes.
2138	*/
2139	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
2140	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
2141	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
2142	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
2143
2144	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
2145	{
2146	if (pGstWalk->u.Amd64.pPte)
2147	{
2148	X86PTEPAE Pte;
2149	Pte.u = pGstWalk->u.Amd64.pPte[1].u;
2150	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2151	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2152	{
2153	pWalk->GCPtr = GCPtr;
2154	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2155	pGstWalk->u.Amd64.Pte.u = Pte.u;
2156	pGstWalk->u.Amd64.pPte++;
2157	return VINF_SUCCESS;
2158	}
2159	}
2160	}
2161	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
2162	{
2163	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2164	if (pGstWalk->u.Amd64.pPde)
2165	{
2166	X86PDEPAE Pde;
2167	Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2168	if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2169	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2170	{
2171	/* Get the new PTE and check out the first entry. */
2172	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2173	&pGstWalk->u.Amd64.pPt);
2174	if (RT_SUCCESS(rc))
2175	{
2176	pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2177	X86PTEPAE Pte;
2178	Pte.u = pGstWalk->u.Amd64.pPte->u;
2179	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2180	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2181	{
2182	pWalk->GCPtr = GCPtr;
2183	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2184	pGstWalk->u.Amd64.Pte.u = Pte.u;
2185	pGstWalk->u.Amd64.Pde.u = Pde.u;
2186	pGstWalk->u.Amd64.pPde++;
2187	return VINF_SUCCESS;
2188	}
2189	}
2190	}
2191	}
2192	}
2193	}
2194	else if (!pWalk->fGigantPage)
2195	{
2196	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2197	{
2198	pWalk->GCPtr = GCPtr;
2199	pWalk->GCPhys += GUEST_PAGE_SIZE;
2200	return VINF_SUCCESS;
2201	}
2202	}
2203	else
2204	{
2205	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2206	{
2207	pWalk->GCPtr = GCPtr;
2208	pWalk->GCPhys += GUEST_PAGE_SIZE;
2209	return VINF_SUCCESS;
2210	}
2211	}
2212	}
2213	}
2214	/* Case we don't handle. Do full walk. */
2215	return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2216	}
2217
2218
2219	/**
2220	* Modify page flags for a range of pages in the guest's tables
2221	*
2222	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2223	*
2224	* @returns VBox status code.
2225	* @param pVCpu The cross context virtual CPU structure.
2226	* @param GCPtr Virtual address of the first page in the range.
2227	* @param cb Size (in bytes) of the range to apply the modification to.
2228	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2229	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2230	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2231	*/
2232	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2233	{
2234	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2235	VMCPU_ASSERT_EMT(pVCpu);
2236
2237	/*
2238	* Validate input.
2239	*/
2240	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2241	Assert(cb);
2242
2243	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2244
2245	/*
2246	* Adjust input.
2247	*/
2248	cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2249	cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2250	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2251
2252	/*
2253	* Call worker.
2254	*/
2255	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2256	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2257	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2258	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2259
2260	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2261	return rc;
2262	}
2263
2264
2265	/**
2266	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2267	*
2268	* @returns @c true if the PDPE is valid, @c false otherwise.
2269	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2270	* @param paPaePdpes The PAE PDPEs to validate.
2271	*
2272	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2273	*/
2274	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2275	{
2276	Assert(paPaePdpes);
2277	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2278	{
2279	X86PDPE const PaePdpe = paPaePdpes[i];
2280	if ( !(PaePdpe.u & X86_PDPE_P)
2281	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2282	{ /* likely */ }
2283	else
2284	return false;
2285	}
2286	return true;
2287	}
2288
2289
2290	/**
2291	* Performs the lazy mapping of the 32-bit guest PD.
2292	*
2293	* @returns VBox status code.
2294	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2295	* @param ppPd Where to return the pointer to the mapping. This is
2296	* always set.
2297	*/
2298	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2299	{
2300	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2301	PGM_LOCK_VOID(pVM);
2302
2303	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2304
2305	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2306	PPGMPAGE pPage;
2307	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2308	if (RT_SUCCESS(rc))
2309	{
2310	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2311	if (RT_SUCCESS(rc))
2312	{
2313	# ifdef IN_RING3
2314	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2315	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2316	# else
2317	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2318	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2319	# endif
2320	PGM_UNLOCK(pVM);
2321	return VINF_SUCCESS;
2322	}
2323	AssertRC(rc);
2324	}
2325	PGM_UNLOCK(pVM);
2326
2327	*ppPd = NULL;
2328	return rc;
2329	}
2330
2331
2332	/**
2333	* Performs the lazy mapping of the PAE guest PDPT.
2334	*
2335	* @returns VBox status code.
2336	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2337	* @param ppPdpt Where to return the pointer to the mapping. This is
2338	* always set.
2339	*/
2340	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2341	{
2342	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2343	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2344	PGM_LOCK_VOID(pVM);
2345
2346	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2347	PPGMPAGE pPage;
2348	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2349	if (RT_SUCCESS(rc))
2350	{
2351	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2352	if (RT_SUCCESS(rc))
2353	{
2354	# ifdef IN_RING3
2355	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2356	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2357	# else
2358	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2359	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2360	# endif
2361	PGM_UNLOCK(pVM);
2362	return VINF_SUCCESS;
2363	}
2364	AssertRC(rc);
2365	}
2366
2367	PGM_UNLOCK(pVM);
2368	*ppPdpt = NULL;
2369	return rc;
2370	}
2371
2372
2373	/**
2374	* Performs the lazy mapping / updating of a PAE guest PD.
2375	*
2376	* @returns Pointer to the mapping.
2377	* @returns VBox status code.
2378	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2379	* @param iPdpt Which PD entry to map (0..3).
2380	* @param ppPd Where to return the pointer to the mapping. This is
2381	* always set.
2382	*/
2383	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2384	{
2385	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2386	PGM_LOCK_VOID(pVM);
2387
2388	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2389	Assert(pGuestPDPT);
2390	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2391	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2392	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2393
2394	PPGMPAGE pPage;
2395	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2396	if (RT_SUCCESS(rc))
2397	{
2398	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2399	AssertRC(rc);
2400	if (RT_SUCCESS(rc))
2401	{
2402	# ifdef IN_RING3
2403	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2404	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2405	# else
2406	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2407	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2408	# endif
2409	if (fChanged)
2410	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2411	PGM_UNLOCK(pVM);
2412	return VINF_SUCCESS;
2413	}
2414	}
2415
2416	/* Invalid page or some failure, invalidate the entry. */
2417	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2418	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2419	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2420
2421	PGM_UNLOCK(pVM);
2422	return rc;
2423	}
2424
2425
2426	/**
2427	* Performs the lazy mapping of the 32-bit guest PD.
2428	*
2429	* @returns VBox status code.
2430	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2431	* @param ppPml4 Where to return the pointer to the mapping. This will
2432	* always be set.
2433	*/
2434	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2435	{
2436	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2437	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2438	PGM_LOCK_VOID(pVM);
2439
2440	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2441	PPGMPAGE pPage;
2442	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2443	if (RT_SUCCESS(rc))
2444	{
2445	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2446	if (RT_SUCCESS(rc))
2447	{
2448	# ifdef IN_RING3
2449	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2450	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2451	# else
2452	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2453	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2454	# endif
2455	PGM_UNLOCK(pVM);
2456	return VINF_SUCCESS;
2457	}
2458	}
2459
2460	PGM_UNLOCK(pVM);
2461	*ppPml4 = NULL;
2462	return rc;
2463	}
2464
2465
2466	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2467	/**
2468	* Performs the lazy mapping of the guest PML4 table when using EPT paging.
2469	*
2470	* @returns VBox status code.
2471	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2472	* @param ppEptPml4 Where to return the pointer to the mapping. This will
2473	* always be set.
2474	*/
2475	int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2476	{
2477	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2478	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2479	PGM_LOCK_VOID(pVM);
2480
2481	RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2482	PPGMPAGE pPage;
2483	int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2484	if (RT_SUCCESS(rc))
2485	{
2486	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2487	if (RT_SUCCESS(rc))
2488	{
2489	# ifdef IN_RING3
2490	pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2491	pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2492	# else
2493	pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2494	pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2495	# endif
2496	PGM_UNLOCK(pVM);
2497	return VINF_SUCCESS;
2498	}
2499	}
2500
2501	PGM_UNLOCK(pVM);
2502	*ppEptPml4 = NULL;
2503	return rc;
2504	}
2505	#endif
2506
2507
2508	/**
2509	* Gets the current CR3 register value for the shadow memory context.
2510	* @returns CR3 value.
2511	* @param pVCpu The cross context virtual CPU structure.
2512	*/
2513	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2514	{
2515	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2516	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2517	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2518	return pPoolPage->Core.Key;
2519	#else
2520	RT_NOREF(pVCpu);
2521	return NIL_RTHCPHYS;
2522	#endif
2523	}
2524
2525
2526	/**
2527	* Forces lazy remapping of the guest's PAE page-directory structures.
2528	*
2529	* @param pVCpu The cross context virtual CPU structure.
2530	*/
2531	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2532	{
2533	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2534	{
2535	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2536	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2537	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2538	}
2539	}
2540
2541
2542	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2543	/**
2544	* Performs second-level address translation for the given CR3 and updates the
2545	* nested-guest CR3 when successful.
2546	*
2547	* @returns VBox status code.
2548	* @param pVCpu The cross context virtual CPU structure.
2549	* @param uCr3 The masked nested-guest CR3 value.
2550	* @param pGCPhysCR3 Where to store the translated CR3.
2551	*
2552	* @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2553	* mindful of this in code that's hyper sensitive to the order of
2554	* operations.
2555	*/
2556	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2557	{
2558	if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2559	{
2560	PGMPTWALK Walk;
2561	PGMPTWALKGST GstWalk;
2562	int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk, &GstWalk);
2563	if (RT_SUCCESS(rc))
2564	{
2565	/* Update nested-guest CR3. */
2566	pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2567
2568	/* Pass back the translated result. */
2569	*pGCPhysCr3 = Walk.GCPhys;
2570	return VINF_SUCCESS;
2571	}
2572
2573	/* Translation failed. */
2574	*pGCPhysCr3 = NIL_RTGCPHYS;
2575	return rc;
2576	}
2577
2578	/*
2579	* If the nested-guest CR3 has not changed, then the previously
2580	* translated CR3 result (i.e. GCPhysCR3) is passed back.
2581	*/
2582	*pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2583	return VINF_SUCCESS;
2584	}
2585	#endif
2586
2587
2588	/**
2589	* Performs and schedules necessary updates following a CR3 load or reload.
2590	*
2591	* This will normally involve mapping the guest PD or nPDPT
2592	*
2593	* @returns VBox status code.
2594	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2595	* safely be ignored and overridden since the FF will be set too then.
2596	* @param pVCpu The cross context virtual CPU structure.
2597	* @param cr3 The new cr3.
2598	* @param fGlobal Indicates whether this is a global flush or not.
2599	*/
2600	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2601	{
2602	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2603	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2604
2605	VMCPU_ASSERT_EMT(pVCpu);
2606
2607	/*
2608	* Always flag the necessary updates; necessary for hardware acceleration
2609	*/
2610	/** @todo optimize this, it shouldn't always be necessary. */
2611	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2612	if (fGlobal)
2613	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2614
2615	/*
2616	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2617	*/
2618	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2619	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2620	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2621	if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2622	&& PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2623	{
2624	RTGCPHYS GCPhysOut;
2625	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2626	if (RT_SUCCESS(rc))
2627	GCPhysCR3 = GCPhysOut;
2628	else
2629	{
2630	/* CR3 SLAT translation failed but we try to pretend it
2631	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2632	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2633	int const rc2 = pgmGstUnmapCr3(pVCpu);
2634	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2635	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2636	return rc2;
2637	}
2638	}
2639	#endif
2640
2641	LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2642	int rc = VINF_SUCCESS;
2643	if (GCPhysOldCR3 != GCPhysCR3)
2644	{
2645	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2646	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2647	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2648
2649	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2650	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2651	if (RT_LIKELY(rc == VINF_SUCCESS))
2652	{ }
2653	else
2654	{
2655	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2656	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2657	pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2658	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2659	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2660	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2661	}
2662
2663	if (fGlobal)
2664	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2665	else
2666	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2667	}
2668	else
2669	{
2670	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2671	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2672	if (pPool->cDirtyPages)
2673	{
2674	PGM_LOCK_VOID(pVM);
2675	pgmPoolResetDirtyPages(pVM);
2676	PGM_UNLOCK(pVM);
2677	}
2678	#endif
2679	if (fGlobal)
2680	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2681	else
2682	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2683
2684	/*
2685	* Flush PAE PDPTEs.
2686	*/
2687	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2688	pgmGstFlushPaePdpes(pVCpu);
2689	}
2690
2691	if (!fGlobal)
2692	IEMTlbInvalidateAll(pVCpu);
2693	else
2694	IEMTlbInvalidateAllGlobal(pVCpu);
2695	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2696	return rc;
2697	}
2698
2699
2700	/**
2701	* Performs and schedules necessary updates following a CR3 load or reload when
2702	* using nested or extended paging.
2703	*
2704	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2705	* TLB and triggering a SyncCR3.
2706	*
2707	* This will normally involve mapping the guest PD or nPDPT
2708	*
2709	* @returns VBox status code.
2710	* @retval VINF_SUCCESS.
2711	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2712	* paging modes). This can safely be ignored and overridden since the
2713	* FF will be set too then.
2714	* @param pVCpu The cross context virtual CPU structure.
2715	* @param cr3 The new CR3.
2716	*/
2717	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2718	{
2719	VMCPU_ASSERT_EMT(pVCpu);
2720
2721	/* We assume we're only called in nested paging mode. */
2722	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2723
2724	/*
2725	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2726	*/
2727	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2728	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2729	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2730	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2731	{
2732	RTGCPHYS GCPhysOut;
2733	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2734	if (RT_SUCCESS(rc))
2735	GCPhysCR3 = GCPhysOut;
2736	else
2737	{
2738	/* CR3 SLAT translation failed but we try to pretend it
2739	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2740	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2741	int const rc2 = pgmGstUnmapCr3(pVCpu);
2742	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2743	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2744	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2745	return rc2;
2746	}
2747	}
2748	#endif
2749
2750	LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2751	int rc = VINF_SUCCESS;
2752	if (GCPhysOldCR3 != GCPhysCR3)
2753	{
2754	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2755	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2756	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2757
2758	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2759	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2760
2761	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2762	}
2763	/*
2764	* Flush PAE PDPTEs.
2765	*/
2766	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2767	pgmGstFlushPaePdpes(pVCpu);
2768
2769	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2770	return rc;
2771	}
2772
2773
2774	/**
2775	* Synchronize the paging structures.
2776	*
2777	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2778	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2779	* in several places, most importantly whenever the CR3 is loaded.
2780	*
2781	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2782	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2783	* the VMM into guest context.
2784	* @param pVCpu The cross context virtual CPU structure.
2785	* @param cr0 Guest context CR0 register
2786	* @param cr3 Guest context CR3 register
2787	* @param cr4 Guest context CR4 register
2788	* @param fGlobal Including global page directories or not
2789	*/
2790	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2791	{
2792	int rc;
2793
2794	VMCPU_ASSERT_EMT(pVCpu);
2795
2796	/*
2797	* The pool may have pending stuff and even require a return to ring-3 to
2798	* clear the whole thing.
2799	*/
2800	rc = pgmPoolSyncCR3(pVCpu);
2801	if (rc != VINF_SUCCESS)
2802	return rc;
2803
2804	/*
2805	* We might be called when we shouldn't.
2806	*
2807	* The mode switching will ensure that the PD is resynced after every mode
2808	* switch. So, if we find ourselves here when in protected or real mode
2809	* we can safely clear the FF and return immediately.
2810	*/
2811	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2812	{
2813	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2814	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2815	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2816	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2817	return VINF_SUCCESS;
2818	}
2819
2820	/* If global pages are not supported, then all flushes are global. */
2821	if (!(cr4 & X86_CR4_PGE))
2822	fGlobal = true;
2823	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2824	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2825
2826	/*
2827	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2828	* This should be done before SyncCR3.
2829	*/
2830	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2831	{
2832	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2833
2834	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2835	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2836	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2837	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2838	{
2839	RTGCPHYS GCPhysOut;
2840	int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2841	if (RT_SUCCESS(rc2))
2842	GCPhysCR3 = GCPhysOut;
2843	else
2844	{
2845	/* CR3 SLAT translation failed but we try to pretend it
2846	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2847	AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2));
2848	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2849	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2850	return rc2;
2851	}
2852	}
2853	#endif
2854	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2855	if (GCPhysOldCR3 != GCPhysCR3)
2856	{
2857	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2858	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2859	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2860	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2861	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2862	}
2863
2864	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2865	if ( rc == VINF_PGM_SYNC_CR3
2866	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2867	{
2868	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2869	#ifdef IN_RING3
2870	rc = pgmPoolSyncCR3(pVCpu);
2871	#else
2872	if (rc == VINF_PGM_SYNC_CR3)
2873	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2874	return VINF_PGM_SYNC_CR3;
2875	#endif
2876	}
2877	AssertRCReturn(rc, rc);
2878	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2879	}
2880
2881	/*
2882	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2883	*/
2884	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2885
2886	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2887	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2888	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2889	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2890
2891	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2892	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2893	if (rc == VINF_SUCCESS)
2894	{
2895	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2896	{
2897	/* Go back to ring 3 if a pgm pool sync is again pending. */
2898	return VINF_PGM_SYNC_CR3;
2899	}
2900
2901	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2902	{
2903	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2904	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2905	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2906	}
2907	}
2908
2909	/*
2910	* Now flush the CR3 (guest context).
2911	*/
2912	if (rc == VINF_SUCCESS)
2913	PGM_INVL_VCPU_TLBS(pVCpu);
2914	return rc;
2915	}
2916
2917
2918	/**
2919	* Maps all the PAE PDPE entries.
2920	*
2921	* @returns VBox status code.
2922	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2923	* @param paPaePdpes The new PAE PDPE values.
2924	*
2925	* @remarks This function may be invoked during the process of changing the guest
2926	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
2927	* reflect PAE paging just yet.
2928	*/
2929	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2930	{
2931	Assert(paPaePdpes);
2932	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2933	{
2934	X86PDPE const PaePdpe = paPaePdpes[i];
2935
2936	/*
2937	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
2938	* are deferred.[1] Also, different situations require different handling of invalid
2939	* PDPE entries. Here we assume the caller has already validated or doesn't require
2940	* validation of the PDPEs.
2941	*
2942	* In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
2943	* validated by the VMX transition.
2944	*
2945	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
2946	*/
2947	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
2948	{
2949	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2950	RTHCPTR HCPtr;
2951
2952	RTGCPHYS GCPhys;
2953	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2954	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2955	{
2956	PGMPTWALK Walk;
2957	PGMPTWALKGST GstWalk;
2958	RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
2959	int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid /, 0 / GCPtrNested */,
2960	&Walk, &GstWalk);
2961	if (RT_SUCCESS(rc))
2962	GCPhys = Walk.GCPhys;
2963	else
2964	{
2965	/*
2966	* Second-level address translation of the PAE PDPE has failed but we must -NOT-
2967	* abort and return a failure now. This is because we're called from a Mov CRx
2968	* instruction (or similar operation). Let's just pretend success but flag that
2969	* we need to map this PDPE lazily later.
2970	*
2971	* See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
2972	* See Intel spec. 28.3.1 "EPT Overview".
2973	*/
2974	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2975	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2976	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2977	continue;
2978	}
2979	}
2980	else
2981	#endif
2982	{
2983	GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
2984	}
2985
2986	PGM_LOCK_VOID(pVM);
2987	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2988	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
2989	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
2990	PGM_UNLOCK(pVM);
2991	if (RT_SUCCESS(rc))
2992	{
2993	#ifdef IN_RING3
2994	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
2995	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
2996	#else
2997	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
2998	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
2999	#endif
3000	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
3001	continue;
3002	}
3003	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
3004	}
3005	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3006	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3007	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3008	}
3009	return VINF_SUCCESS;
3010	}
3011
3012
3013	/**
3014	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
3015	*
3016	* @returns VBox status code.
3017	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3018	* @param cr3 The guest CR3 value.
3019	*
3020	* @remarks This function may be invoked during the process of changing the guest
3021	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
3022	* PAE paging just yet.
3023	*/
3024	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
3025	{
3026	/*
3027	* Read the page-directory-pointer table (PDPT) at CR3.
3028	*/
3029	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
3030	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
3031
3032	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3033	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3034	{
3035	RTGCPHYS GCPhysOut;
3036	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
3037	if (RT_SUCCESS(rc))
3038	GCPhysCR3 = GCPhysOut;
3039	else
3040	{
3041	Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
3042	return rc;
3043	}
3044	}
3045	#endif
3046
3047	RTHCPTR HCPtrGuestCr3;
3048	int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
3049	if (RT_SUCCESS(rc))
3050	{
3051	/*
3052	* Validate the page-directory-pointer table entries (PDPE).
3053	*/
3054	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
3055	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
3056	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
3057	{
3058	/*
3059	* Map the PDPT.
3060	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
3061	* that PGMFlushTLB will be called soon and only a change to CR3 then
3062	* will cause the shadow page tables to be updated.
3063	*/
3064	#ifdef IN_RING3
3065	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
3066	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
3067	#else
3068	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
3069	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
3070	#endif
3071
3072	/*
3073	* Update CPUM and map the 4 PAE PDPEs.
3074	*/
3075	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
3076	rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
3077	if (RT_SUCCESS(rc))
3078	{
3079	#ifdef IN_RING3
3080	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
3081	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
3082	#else
3083	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
3084	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
3085	#endif
3086	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
3087	}
3088	}
3089	else
3090	rc = VERR_PGM_PAE_PDPE_RSVD;
3091	}
3092	return rc;
3093	}
3094
3095
3096	/**
3097	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
3098	*
3099	* @returns VBox status code, with the following informational code for
3100	* VM scheduling.
3101	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
3102	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
3103	*
3104	* @param pVCpu The cross context virtual CPU structure.
3105	* @param cr0 The new cr0.
3106	* @param cr4 The new cr4.
3107	* @param efer The new extended feature enable register.
3108	* @param fForce Whether to force a mode change.
3109	*/
3110	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
3111	{
3112	VMCPU_ASSERT_EMT(pVCpu);
3113
3114	/*
3115	* Calc the new guest mode.
3116	*
3117	* Note! We check PG before PE and without requiring PE because of the
3118	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
3119	*/
3120	PGMMODE enmGuestMode;
3121	if (cr0 & X86_CR0_PG)
3122	{
3123	if (!(cr4 & X86_CR4_PAE))
3124	{
3125	bool const fPse = !!(cr4 & X86_CR4_PSE);
3126	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
3127	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
3128	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
3129	enmGuestMode = PGMMODE_32_BIT;
3130	}
3131	else if (!(efer & MSR_K6_EFER_LME))
3132	{
3133	if (!(efer & MSR_K6_EFER_NXE))
3134	enmGuestMode = PGMMODE_PAE;
3135	else
3136	enmGuestMode = PGMMODE_PAE_NX;
3137	}
3138	else
3139	{
3140	if (!(efer & MSR_K6_EFER_NXE))
3141	enmGuestMode = PGMMODE_AMD64;
3142	else
3143	enmGuestMode = PGMMODE_AMD64_NX;
3144	}
3145	}
3146	else if (!(cr0 & X86_CR0_PE))
3147	enmGuestMode = PGMMODE_REAL;
3148	else
3149	enmGuestMode = PGMMODE_PROTECTED;
3150
3151	/*
3152	* Did it change?
3153	*/
3154	if ( !fForce
3155	&& pVCpu->pgm.s.enmGuestMode == enmGuestMode)
3156	return VINF_SUCCESS;
3157
3158	/* Flush the TLB */
3159	PGM_INVL_VCPU_TLBS(pVCpu);
3160	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3161	}
3162
3163
3164	/**
3165	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3166	*
3167	* @returns PGM_TYPE_*.
3168	* @param pgmMode The mode value to convert.
3169	*/
3170	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3171	{
3172	switch (pgmMode)
3173	{
3174	case PGMMODE_REAL: return PGM_TYPE_REAL;
3175	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3176	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3177	case PGMMODE_PAE:
3178	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3179	case PGMMODE_AMD64:
3180	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3181	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3182	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3183	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3184	case PGMMODE_EPT: return PGM_TYPE_EPT;
3185	case PGMMODE_NONE: return PGM_TYPE_NONE;
3186	default:
3187	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3188	}
3189	}
3190
3191
3192	/**
3193	* Calculates the shadow paging mode.
3194	*
3195	* @returns The shadow paging mode.
3196	* @param pVM The cross context VM structure.
3197	* @param enmGuestMode The guest mode.
3198	* @param enmHostMode The host mode.
3199	* @param enmShadowMode The current shadow mode.
3200	*/
3201	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3202	{
3203	switch (enmGuestMode)
3204	{
3205	case PGMMODE_REAL:
3206	case PGMMODE_PROTECTED:
3207	switch (enmHostMode)
3208	{
3209	case SUPPAGINGMODE_32_BIT:
3210	case SUPPAGINGMODE_32_BIT_GLOBAL:
3211	enmShadowMode = PGMMODE_32_BIT;
3212	break;
3213
3214	case SUPPAGINGMODE_PAE:
3215	case SUPPAGINGMODE_PAE_NX:
3216	case SUPPAGINGMODE_PAE_GLOBAL:
3217	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3218	enmShadowMode = PGMMODE_PAE;
3219	break;
3220
3221	case SUPPAGINGMODE_AMD64:
3222	case SUPPAGINGMODE_AMD64_GLOBAL:
3223	case SUPPAGINGMODE_AMD64_NX:
3224	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3225	enmShadowMode = PGMMODE_PAE;
3226	break;
3227
3228	default:
3229	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3230	}
3231	break;
3232
3233	case PGMMODE_32_BIT:
3234	switch (enmHostMode)
3235	{
3236	case SUPPAGINGMODE_32_BIT:
3237	case SUPPAGINGMODE_32_BIT_GLOBAL:
3238	enmShadowMode = PGMMODE_32_BIT;
3239	break;
3240
3241	case SUPPAGINGMODE_PAE:
3242	case SUPPAGINGMODE_PAE_NX:
3243	case SUPPAGINGMODE_PAE_GLOBAL:
3244	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3245	enmShadowMode = PGMMODE_PAE;
3246	break;
3247
3248	case SUPPAGINGMODE_AMD64:
3249	case SUPPAGINGMODE_AMD64_GLOBAL:
3250	case SUPPAGINGMODE_AMD64_NX:
3251	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3252	enmShadowMode = PGMMODE_PAE;
3253	break;
3254
3255	default:
3256	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3257	}
3258	break;
3259
3260	case PGMMODE_PAE:
3261	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3262	switch (enmHostMode)
3263	{
3264	case SUPPAGINGMODE_32_BIT:
3265	case SUPPAGINGMODE_32_BIT_GLOBAL:
3266	enmShadowMode = PGMMODE_PAE;
3267	break;
3268
3269	case SUPPAGINGMODE_PAE:
3270	case SUPPAGINGMODE_PAE_NX:
3271	case SUPPAGINGMODE_PAE_GLOBAL:
3272	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3273	enmShadowMode = PGMMODE_PAE;
3274	break;
3275
3276	case SUPPAGINGMODE_AMD64:
3277	case SUPPAGINGMODE_AMD64_GLOBAL:
3278	case SUPPAGINGMODE_AMD64_NX:
3279	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3280	enmShadowMode = PGMMODE_PAE;
3281	break;
3282
3283	default:
3284	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3285	}
3286	break;
3287
3288	case PGMMODE_AMD64:
3289	case PGMMODE_AMD64_NX:
3290	switch (enmHostMode)
3291	{
3292	case SUPPAGINGMODE_32_BIT:
3293	case SUPPAGINGMODE_32_BIT_GLOBAL:
3294	enmShadowMode = PGMMODE_AMD64;
3295	break;
3296
3297	case SUPPAGINGMODE_PAE:
3298	case SUPPAGINGMODE_PAE_NX:
3299	case SUPPAGINGMODE_PAE_GLOBAL:
3300	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3301	enmShadowMode = PGMMODE_AMD64;
3302	break;
3303
3304	case SUPPAGINGMODE_AMD64:
3305	case SUPPAGINGMODE_AMD64_GLOBAL:
3306	case SUPPAGINGMODE_AMD64_NX:
3307	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3308	enmShadowMode = PGMMODE_AMD64;
3309	break;
3310
3311	default:
3312	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3313	}
3314	break;
3315
3316	default:
3317	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3318	}
3319
3320	/*
3321	* Override the shadow mode when NEM, IEM or nested paging is active.
3322	*/
3323	if (!VM_IS_HM_ENABLED(pVM))
3324	{
3325	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
3326	pVM->pgm.s.fNestedPaging = true;
3327	enmShadowMode = PGMMODE_NONE;
3328	}
3329	else
3330	{
3331	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3332	pVM->pgm.s.fNestedPaging = fNestedPaging;
3333	if (fNestedPaging)
3334	{
3335	if (HMIsVmxActive(pVM))
3336	enmShadowMode = PGMMODE_EPT;
3337	else
3338	{
3339	/* The nested SVM paging depends on the host one. */
3340	Assert(HMIsSvmActive(pVM));
3341	if ( enmGuestMode == PGMMODE_AMD64
3342	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3343	enmShadowMode = PGMMODE_NESTED_AMD64;
3344	else
3345	switch (pVM->pgm.s.enmHostMode)
3346	{
3347	case SUPPAGINGMODE_32_BIT:
3348	case SUPPAGINGMODE_32_BIT_GLOBAL:
3349	enmShadowMode = PGMMODE_NESTED_32BIT;
3350	break;
3351
3352	case SUPPAGINGMODE_PAE:
3353	case SUPPAGINGMODE_PAE_GLOBAL:
3354	case SUPPAGINGMODE_PAE_NX:
3355	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3356	enmShadowMode = PGMMODE_NESTED_PAE;
3357	break;
3358
3359	case SUPPAGINGMODE_AMD64:
3360	case SUPPAGINGMODE_AMD64_GLOBAL:
3361	case SUPPAGINGMODE_AMD64_NX:
3362	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3363	enmShadowMode = PGMMODE_NESTED_AMD64;
3364	break;
3365
3366	default:
3367	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3368	}
3369	}
3370	}
3371	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3372	else
3373	{
3374	/* Nested paging is a requirement for nested VT-x. */
3375	AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3376	}
3377	#endif
3378	}
3379
3380	return enmShadowMode;
3381	}
3382
3383
3384	/**
3385	* Performs the actual mode change.
3386	* This is called by PGMChangeMode and pgmR3InitPaging().
3387	*
3388	* @returns VBox status code. May suspend or power off the VM on error, but this
3389	* will trigger using FFs and not informational status codes.
3390	*
3391	* @param pVM The cross context VM structure.
3392	* @param pVCpu The cross context virtual CPU structure.
3393	* @param enmGuestMode The new guest mode. This is assumed to be different from
3394	* the current mode.
3395	* @param fForce Whether to force a shadow paging mode change.
3396	*/
3397	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3398	{
3399	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3400	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3401
3402	/*
3403	* Calc the shadow mode and switcher.
3404	*/
3405	PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3406	bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode \|\| fForce;
3407
3408	/*
3409	* Exit old mode(s).
3410	*/
3411	/* shadow */
3412	if (fShadowModeChanged)
3413	{
3414	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3415	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3416	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3417	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3418	{
3419	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3420	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3421	}
3422	}
3423	else
3424	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3425
3426	/* guest */
3427	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3428	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3429	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3430	{
3431	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3432	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3433	}
3434	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3435	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3436	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3437	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3438
3439	/*
3440	* Change the paging mode data indexes.
3441	*/
3442	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3443	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3444	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3445	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3446	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3447	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3448	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3449	#ifdef IN_RING3
3450	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3451	#endif
3452
3453	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3454	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3455	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3456	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3457	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3458	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3459	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3460	#ifdef IN_RING3
3461	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3462	#endif
3463
3464	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3465	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3466	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3467	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3468	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3469	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3470	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3471	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3472	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3473	#ifdef VBOX_STRICT
3474	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3475	#endif
3476
3477	/*
3478	* Determine SLAT mode -before- entering the new shadow mode!
3479	*/
3480	pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
3481
3482	/*
3483	* Enter new shadow mode (if changed).
3484	*/
3485	if (fShadowModeChanged)
3486	{
3487	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3488	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
3489	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3490	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3491	}
3492
3493	/*
3494	* Always flag the necessary updates
3495	*/
3496	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3497
3498	/*
3499	* Enter the new guest and shadow+guest modes.
3500	*/
3501	/* Calc the new CR3 value. */
3502	RTGCPHYS GCPhysCR3;
3503	switch (enmGuestMode)
3504	{
3505	case PGMMODE_REAL:
3506	case PGMMODE_PROTECTED:
3507	GCPhysCR3 = NIL_RTGCPHYS;
3508	break;
3509
3510	case PGMMODE_32_BIT:
3511	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3512	break;
3513
3514	case PGMMODE_PAE_NX:
3515	case PGMMODE_PAE:
3516	if (!pVM->cpum.ro.GuestFeatures.fPae)
3517	#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3518	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3519	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3520	#else
3521	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3522
3523	#endif
3524	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3525	break;
3526
3527	#ifdef VBOX_WITH_64_BITS_GUESTS
3528	case PGMMODE_AMD64_NX:
3529	case PGMMODE_AMD64:
3530	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3531	break;
3532	#endif
3533	default:
3534	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3535	}
3536
3537	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3538	/*
3539	* If a nested-guest is using EPT paging:
3540	* - Update the second-level address translation (SLAT) mode.
3541	* - Indicate that the CR3 is nested-guest physical address.
3542	*/
3543	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3544	{
3545	if (PGMMODE_WITH_PAGING(enmGuestMode))
3546	{
3547	/*
3548	* Translate CR3 to its guest-physical address.
3549	* We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3550	* switching modes to keep it consistent with how GCPhysCR3 is updated.
3551	*/
3552	PGMPTWALK Walk;
3553	PGMPTWALKGST GstWalk;
3554	int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk,
3555	&GstWalk);
3556	if (RT_SUCCESS(rc))
3557	{ /* likely */ }
3558	else
3559	{
3560	/*
3561	* SLAT failed but we avoid reporting this to the caller because the caller
3562	* is not supposed to fail. The only time the caller needs to indicate a
3563	* failure to software is when PAE paging is used by the nested-guest, but
3564	* we handle the PAE case separately (e.g., see VMX transition in IEM).
3565	* In all other cases, the failure will be indicated when CR3 tries to be
3566	* translated on the next linear-address memory access.
3567	* See Intel spec. 27.2.1 "EPT Overview".
3568	*/
3569	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3570
3571	/* Trying to coax PGM to succeed for the time being... */
3572	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3573	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3574	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3575	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3576	return VINF_SUCCESS;
3577	}
3578	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3579	GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
3580	}
3581	}
3582	else
3583	Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3584	#endif
3585
3586	/*
3587	* Enter the new guest mode.
3588	*/
3589	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3590	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3591	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3592
3593	/* Set the new guest CR3 (and nested-guest CR3). */
3594	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3595
3596	/* status codes. */
3597	AssertRC(rc);
3598	AssertRC(rc2);
3599	if (RT_SUCCESS(rc))
3600	{
3601	rc = rc2;
3602	if (RT_SUCCESS(rc)) /* no informational status codes. */
3603	rc = VINF_SUCCESS;
3604	}
3605
3606	/*
3607	* Notify HM.
3608	*/
3609	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3610	return rc;
3611	}
3612
3613
3614	/**
3615	* Called by CPUM or REM when CR0.WP changes to 1.
3616	*
3617	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3618	* @thread EMT
3619	*/
3620	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3621	{
3622	/*
3623	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3624	*
3625	* Use the counter to judge whether there might be pool pages with active
3626	* hacks in them. If there are, we will be running the risk of messing up
3627	* the guest by allowing it to write to read-only pages. Thus, we have to
3628	* clear the page pool ASAP if there is the slightest chance.
3629	*/
3630	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3631	{
3632	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3633
3634	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3635	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3636	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3637	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3638	}
3639	}
3640
3641
3642	/**
3643	* Gets the current guest paging mode.
3644	*
3645	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3646	*
3647	* @returns The current paging mode.
3648	* @param pVCpu The cross context virtual CPU structure.
3649	*/
3650	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3651	{
3652	return pVCpu->pgm.s.enmGuestMode;
3653	}
3654
3655
3656	/**
3657	* Gets the current shadow paging mode.
3658	*
3659	* @returns The current paging mode.
3660	* @param pVCpu The cross context virtual CPU structure.
3661	*/
3662	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3663	{
3664	return pVCpu->pgm.s.enmShadowMode;
3665	}
3666
3667
3668	/**
3669	* Gets the current host paging mode.
3670	*
3671	* @returns The current paging mode.
3672	* @param pVM The cross context VM structure.
3673	*/
3674	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3675	{
3676	switch (pVM->pgm.s.enmHostMode)
3677	{
3678	case SUPPAGINGMODE_32_BIT:
3679	case SUPPAGINGMODE_32_BIT_GLOBAL:
3680	return PGMMODE_32_BIT;
3681
3682	case SUPPAGINGMODE_PAE:
3683	case SUPPAGINGMODE_PAE_GLOBAL:
3684	return PGMMODE_PAE;
3685
3686	case SUPPAGINGMODE_PAE_NX:
3687	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3688	return PGMMODE_PAE_NX;
3689
3690	case SUPPAGINGMODE_AMD64:
3691	case SUPPAGINGMODE_AMD64_GLOBAL:
3692	return PGMMODE_AMD64;
3693
3694	case SUPPAGINGMODE_AMD64_NX:
3695	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3696	return PGMMODE_AMD64_NX;
3697
3698	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3699	}
3700
3701	return PGMMODE_INVALID;
3702	}
3703
3704
3705	/**
3706	* Get mode name.
3707	*
3708	* @returns read-only name string.
3709	* @param enmMode The mode which name is desired.
3710	*/
3711	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3712	{
3713	switch (enmMode)
3714	{
3715	case PGMMODE_REAL: return "Real";
3716	case PGMMODE_PROTECTED: return "Protected";
3717	case PGMMODE_32_BIT: return "32-bit";
3718	case PGMMODE_PAE: return "PAE";
3719	case PGMMODE_PAE_NX: return "PAE+NX";
3720	case PGMMODE_AMD64: return "AMD64";
3721	case PGMMODE_AMD64_NX: return "AMD64+NX";
3722	case PGMMODE_NESTED_32BIT: return "Nested-32";
3723	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3724	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3725	case PGMMODE_EPT: return "EPT";
3726	case PGMMODE_NONE: return "None";
3727	default: return "unknown mode value";
3728	}
3729	}
3730
3731
3732	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3733	/**
3734	* Gets the SLAT mode name.
3735	*
3736	* @returns The read-only SLAT mode descriptive string.
3737	* @param enmSlatMode The SLAT mode value.
3738	*/
3739	VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3740	{
3741	switch (enmSlatMode)
3742	{
3743	case PGMSLAT_DIRECT: return "Direct";
3744	case PGMSLAT_EPT: return "EPT";
3745	case PGMSLAT_32BIT: return "32-bit";
3746	case PGMSLAT_PAE: return "PAE";
3747	case PGMSLAT_AMD64: return "AMD64";
3748	default: return "Unknown";
3749	}
3750	}
3751	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3752
3753
3754	/**
3755	* Gets the physical address represented in the guest CR3 as PGM sees it.
3756	*
3757	* This is mainly for logging and debugging.
3758	*
3759	* @returns PGM's guest CR3 value.
3760	* @param pVCpu The cross context virtual CPU structure.
3761	*/
3762	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3763	{
3764	return pVCpu->pgm.s.GCPhysCR3;
3765	}
3766
3767
3768
3769	/**
3770	* Notification from CPUM that the EFER.NXE bit has changed.
3771	*
3772	* @param pVCpu The cross context virtual CPU structure of the CPU for
3773	* which EFER changed.
3774	* @param fNxe The new NXE state.
3775	*/
3776	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3777	{
3778	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3779	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3780
3781	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3782	if (fNxe)
3783	{
3784	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3785	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3786	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3787	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3788	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3789	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3790	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3791	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3792	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3793	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3794	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3795
3796	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3797	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3798	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3799	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3800	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3801	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3802	}
3803	else
3804	{
3805	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3806	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3807	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3808	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3809	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3810	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3811	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3812	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3813	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3814	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3815	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3816
3817	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3818	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3819	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3820	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3821	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3822	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3823	}
3824	}
3825
3826
3827	/**
3828	* Check if any pgm pool pages are marked dirty (not monitored)
3829	*
3830	* @returns bool locked/not locked
3831	* @param pVM The cross context VM structure.
3832	*/
3833	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3834	{
3835	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3836	}
3837
3838
3839	/**
3840	* Check if this VCPU currently owns the PGM lock.
3841	*
3842	* @returns bool owner/not owner
3843	* @param pVM The cross context VM structure.
3844	*/
3845	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3846	{
3847	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3848	}
3849
3850
3851	/**
3852	* Enable or disable large page usage
3853	*
3854	* @returns VBox status code.
3855	* @param pVM The cross context VM structure.
3856	* @param fUseLargePages Use/not use large pages
3857	*/
3858	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3859	{
3860	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3861
3862	pVM->pgm.s.fUseLargePages = fUseLargePages;
3863	return VINF_SUCCESS;
3864	}
3865
3866
3867	/**
3868	* Acquire the PGM lock.
3869	*
3870	* @returns VBox status code
3871	* @param pVM The cross context VM structure.
3872	* @param fVoid Set if the caller cannot handle failure returns.
3873	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3874	*/
3875	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3876	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3877	#else
3878	int pgmLock(PVMCC pVM, bool fVoid)
3879	#endif
3880	{
3881	#if defined(VBOX_STRICT)
3882	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3883	#else
3884	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3885	#endif
3886	if (RT_SUCCESS(rc))
3887	return rc;
3888	if (fVoid)
3889	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3890	else
3891	AssertRC(rc);
3892	return rc;
3893	}
3894
3895
3896	/**
3897	* Release the PGM lock.
3898	*
3899	* @param pVM The cross context VM structure.
3900	*/
3901	void pgmUnlock(PVMCC pVM)
3902	{
3903	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3904	pVM->pgm.s.cDeprecatedPageLocks = 0;
3905	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3906	if (rc == VINF_SEM_NESTED)
3907	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3908	}
3909
3910
3911	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3912
3913	/** Format handler for PGMPAGE.
3914	* @copydoc FNRTSTRFORMATTYPE */
3915	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3916	const char pszType, void const pvValue,
3917	int cchWidth, int cchPrecision, unsigned fFlags,
3918	void *pvUser)
3919	{
3920	size_t cch;
3921	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3922	if (RT_VALID_PTR(pPage))
3923	{
3924	char szTmp[64+80];
3925
3926	cch = 0;
3927
3928	/* The single char state stuff. */
3929	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3930	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3931
3932	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3933	if (IS_PART_INCLUDED(5))
3934	{
3935	static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
3936	szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
3937	\| ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
3938	}
3939
3940	/* The type. */
3941	if (IS_PART_INCLUDED(4))
3942	{
3943	szTmp[cch++] = ':';
3944	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3945	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3946	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3947	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3948	}
3949
3950	/* The numbers. */
3951	if (IS_PART_INCLUDED(3))
3952	{
3953	szTmp[cch++] = ':';
3954	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3955	}
3956
3957	if (IS_PART_INCLUDED(2))
3958	{
3959	szTmp[cch++] = ':';
3960	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3961	}
3962
3963	if (IS_PART_INCLUDED(6))
3964	{
3965	szTmp[cch++] = ':';
3966	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3967	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3968	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3969	}
3970	# undef IS_PART_INCLUDED
3971
3972	cch = pfnOutput(pvArgOutput, szTmp, cch);
3973	#if 0
3974	size_t cch2 = 0;
3975	szTmp[cch2++] = '(';
3976	cch2 += RTStrFormatNumber(&szTmp[cch2], (uintptr_t)pPage, 16, 18, 0, RTSTR_F_SPECIAL \| RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3977	szTmp[cch2++] = ')';
3978	szTmp[cch2] = '\0';
3979	cch += pfnOutput(pvArgOutput, szTmp, cch2);
3980	#endif
3981	}
3982	else
3983	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3984	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3985	return cch;
3986	}
3987
3988
3989	/** Format handler for PGMRAMRANGE.
3990	* @copydoc FNRTSTRFORMATTYPE */
3991	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3992	const char pszType, void const pvValue,
3993	int cchWidth, int cchPrecision, unsigned fFlags,
3994	void *pvUser)
3995	{
3996	size_t cch;
3997	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3998	if (RT_VALID_PTR(pRam))
3999	{
4000	char szTmp[80];
4001	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
4002	cch = pfnOutput(pvArgOutput, szTmp, cch);
4003	}
4004	else
4005	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
4006	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
4007	return cch;
4008	}
4009
4010	/** Format type andlers to be registered/deregistered. */
4011	static const struct
4012	{
4013	char szType[24];
4014	PFNRTSTRFORMATTYPE pfnHandler;
4015	} g_aPgmFormatTypes[] =
4016	{
4017	{ "pgmpage", pgmFormatTypeHandlerPage },
4018	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
4019	};
4020
4021	#endif /* !IN_R0 \|\| LOG_ENABLED */
4022
4023	/**
4024	* Registers the global string format types.
4025	*
4026	* This should be called at module load time or in some other manner that ensure
4027	* that it's called exactly one time.
4028	*
4029	* @returns IPRT status code on RTStrFormatTypeRegister failure.
4030	*/
4031	VMMDECL(int) PGMRegisterStringFormatTypes(void)
4032	{
4033	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4034	int rc = VINF_SUCCESS;
4035	unsigned i;
4036	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4037	{
4038	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4039	# ifdef IN_RING0
4040	if (rc == VERR_ALREADY_EXISTS)
4041	{
4042	/* in case of cleanup failure in ring-0 */
4043	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4044	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4045	}
4046	# endif
4047	}
4048	if (RT_FAILURE(rc))
4049	while (i-- > 0)
4050	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4051
4052	return rc;
4053	#else
4054	return VINF_SUCCESS;
4055	#endif
4056	}
4057
4058
4059	/**
4060	* Deregisters the global string format types.
4061	*
4062	* This should be called at module unload time or in some other manner that
4063	* ensure that it's called exactly one time.
4064	*/
4065	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
4066	{
4067	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4068	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4069	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4070	#endif
4071	}
4072
4073
4074	#ifdef VBOX_STRICT
4075	/**
4076	* Asserts that everything related to the guest CR3 is correctly shadowed.
4077	*
4078	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
4079	* and assert the correctness of the guest CR3 mapping before asserting that the
4080	* shadow page tables is in sync with the guest page tables.
4081	*
4082	* @returns Number of conflicts.
4083	* @param pVM The cross context VM structure.
4084	* @param pVCpu The cross context virtual CPU structure.
4085	* @param cr3 The current guest CR3 register value.
4086	* @param cr4 The current guest CR4 register value.
4087	*/
4088	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
4089	{
4090	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4091
4092	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
4093	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
4094	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
4095
4096	PGM_LOCK_VOID(pVM);
4097	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
4098	PGM_UNLOCK(pVM);
4099
4100	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4101	return cErrors;
4102	}
4103	#endif /* VBOX_STRICT */
4104
4105
4106	/**
4107	* Updates PGM's copy of the guest's EPT pointer.
4108	*
4109	* @param pVCpu The cross context virtual CPU structure.
4110	* @param uEptPtr The EPT pointer.
4111	*
4112	* @remarks This can be called as part of VM-entry so we might be in the midst of
4113	* switching to VMX non-root mode.
4114	*/
4115	VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
4116	{
4117	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4118	PGM_LOCK_VOID(pVM);
4119	pVCpu->pgm.s.uEptPtr = uEptPtr;
4120	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4121	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4122	PGM_UNLOCK(pVM);
4123	}
4124
4125	#ifdef PGM_WITH_PAGE_ZEROING_DETECTION
4126
4127	/**
4128	* Helper for checking whether XMM0 is zero, possibly retriving external state.
4129	*/
4130	static bool pgmHandlePageZeroingIsXmm0Zero(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4131	{
4132	if (pCtx->fExtrn & CPUMCTX_EXTRN_SSE_AVX)
4133	{
4134	int rc = CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_SSE_AVX);
4135	AssertRCReturn(rc, false);
4136	}
4137	return pCtx->XState.x87.aXMM[0].au64[0] == 0
4138	&& pCtx->XState.x87.aXMM[0].au64[1] == 0
4139	&& pCtx->XState.x87.aXMM[0].au64[2] == 0
4140	&& pCtx->XState.x87.aXMM[0].au64[3] == 0;
4141	}
4142
4143
4144	/**
4145	* Helper for comparing opcode bytes.
4146	*/
4147	static bool pgmHandlePageZeroingMatchOpcodes(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t const *pbOpcodes, uint32_t cbOpcodes)
4148	{
4149	uint8_t abTmp[64];
4150	AssertMsgReturn(cbOpcodes <= sizeof(abTmp), ("cbOpcodes=%#x\n", cbOpcodes), false);
4151	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abTmp, pCtx->rip + pCtx->cs.u64Base, cbOpcodes);
4152	if (RT_SUCCESS(rc))
4153	return memcmp(abTmp, pbOpcodes, cbOpcodes) == 0;
4154	return false;
4155	}
4156
4157
4158	/**
4159	* Called on faults on ZERO pages to check if the guest is trying to zero it.
4160	*
4161	* Since it's a waste of time to zero a ZERO page and it will cause an
4162	* unnecessary page allocation, we'd like to detect and avoid this.
4163	* If any known page zeroing code is detected, this function will update the CPU
4164	* state to pretend the page was zeroed by the code.
4165	*
4166	* @returns true if page zeroing code was detected and CPU state updated to skip
4167	* the code.
4168	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4169	* @param pCtx The guest register context.
4170	*/
4171	static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4172	{
4173	CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
4174
4175	/*
4176	* Sort by mode first.
4177	*/
4178	if (CPUMIsGuestInLongModeEx(pCtx))
4179	{
4180	if (CPUMIsGuestIn64BitCodeEx(pCtx))
4181	{
4182	/*
4183	* 64-bit code.
4184	*/
4185	Log9(("pgmHandlePageZeroingCode: not page zeroing - 64-bit\n"));
4186	}
4187	else if (pCtx->cs.Attr.n.u1DefBig)
4188	Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit lm\n"));
4189	else
4190	Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit lm\n"));
4191	}
4192	else if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
4193	{
4194	if (pCtx->cs.Attr.n.u1DefBig)
4195	{
4196	/*
4197	* 32-bit paged protected mode code.
4198	*/
4199	CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX
4200	\| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP \| CPUMCTX_EXTRN_RSI \| CPUMCTX_EXTRN_RDI
4201	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4202
4203	/* 1. Generic 'rep stosd' detection. */
4204	static uint8_t const s_abRepStosD[] = { 0xf3, 0xab };
4205	if ( pCtx->eax == 0
4206	&& pCtx->ecx == X86_PAGE_SIZE / 4
4207	&& !(pCtx->edi & X86_PAGE_OFFSET_MASK)
4208	&& pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abRepStosD, sizeof(s_abRepStosD)))
4209	{
4210	pCtx->ecx = 0;
4211	pCtx->edi += X86_PAGE_SIZE;
4212	Log9(("pgmHandlePageZeroingCode: REP STOSD: eip=%RX32 -> %RX32\n", pCtx->eip, pCtx->eip + sizeof(s_abRepStosD)));
4213	pCtx->eip += sizeof(s_abRepStosD);
4214	return true;
4215	}
4216
4217	/* 2. Windows 2000 sp4 KiXMMIZeroPageNoSave loop code: */
4218	static uint8_t const s_abW2kSp4XmmZero[] =
4219	{
4220	0x0f, 0x2b, 0x01,
4221	0x0f, 0x2b, 0x41, 0x10,
4222	0x0f, 0x2b, 0x41, 0x20,
4223	0x0f, 0x2b, 0x41, 0x30,
4224	0x83, 0xc1, 0x40,
4225	0x48,
4226	0x75, 0xeb,
4227	};
4228	if ( pCtx->eax == 64
4229	&& !(pCtx->ecx & X86_PAGE_OFFSET_MASK)
4230	&& pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abW2kSp4XmmZero, sizeof(s_abW2kSp4XmmZero))
4231	&& pgmHandlePageZeroingIsXmm0Zero(pVCpu, pCtx))
4232	{
4233	pCtx->eax = 1;
4234	pCtx->ecx += X86_PAGE_SIZE;
4235	Log9(("pgmHandlePageZeroingCode: w2k sp4 xmm: eip=%RX32 -> %RX32\n",
4236	pCtx->eip, pCtx->eip + sizeof(s_abW2kSp4XmmZero) - 3));
4237	pCtx->eip += sizeof(s_abW2kSp4XmmZero) - 3;
4238	return true;
4239	}
4240	Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit\n"));
4241	}
4242	else if (!pCtx->eflags.Bits.u1VM)
4243	Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit\n"));
4244	else
4245	Log9(("pgmHandlePageZeroingCode: not page zeroing - v86\n"));
4246	}
4247	return false;
4248	}
4249
4250	#endif /* PGM_WITH_PAGE_ZEROING_DETECTION */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 107631

Download in other formats: