PGMAll.cpp@ 108132

Last change on this file since 108132 was 108132, checked in by vboxsync, 11 days ago
VMM/PGM: Merge and deduplicate code targeting x86 & amd64 in PGM.cpp. Don't bother compiling pool stuff on arm and darwin.amd64. jiraref:VBP-1531
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 157.0 KB

Line
1	/* $Id: PGMAll.cpp 108132 2025-02-10 11:05:23Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#ifdef IN_RING0
35	# define VBOX_VMM_TARGET_X86
36	#endif
37	#include <VBox/vmm/pgm.h>
38	#include <VBox/vmm/cpum.h>
39	#include <VBox/vmm/selm.h>
40	#include <VBox/vmm/iem.h>
41	#include <VBox/vmm/iom.h>
42	#include <VBox/sup.h>
43	#include <VBox/vmm/mm.h>
44	#include <VBox/vmm/stam.h>
45	#include <VBox/vmm/trpm.h>
46	#include <VBox/vmm/em.h>
47	#include <VBox/vmm/hm.h>
48	#include <VBox/vmm/hm_vmx.h>
49	#include "PGMInternal.h"
50	#include <VBox/vmm/vmcc.h>
51	#include "PGMInline.h"
52	#include <iprt/assert.h>
53	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
54	# include <iprt/asm-amd64-x86.h>
55	#endif
56	#include <iprt/string.h>
57	#include <VBox/log.h>
58	#include <VBox/param.h>
59	#include <VBox/err.h>
60
61
62	/*********************************************************************************************************************************
63	* Internal Functions *
64	*********************************************************************************************************************************/
65	#ifdef VBOX_VMM_TARGET_X86
66	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
67	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
68	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
69	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
70	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
71	PPGMPTWALKGST pGstWalk);
72	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
73	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
74	PPGMPTWALKGST pGstWalkAll);
75	# endif
76	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
77	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
78	# ifdef PGM_WITH_PAGE_ZEROING_DETECTION
79	static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx);
80	# endif
81	#endif /* VBOX_VMM_TARGET_X86 */
82
83
84	#ifdef VBOX_VMM_TARGET_X86
85
86	/*
87	* Second level transation - EPT.
88	*/
89	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
90	# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
91	# include "PGMSlatDefs.h"
92	# include "PGMAllGstSlatEpt.cpp.h"
93	# undef PGM_SLAT_TYPE
94	# endif
95
96
97	/*
98	* Shadow - 32-bit mode
99	*/
100	# define PGM_SHW_TYPE PGM_TYPE_32BIT
101	# define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
102	# include "PGMAllShw.h"
103
104	/* Guest - real mode */
105	# define PGM_GST_TYPE PGM_TYPE_REAL
106	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
107	# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
108	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
109	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
110	# include "PGMGstDefs.h"
111	# include "PGMAllGst.h"
112	# include "PGMAllBth.h"
113	# undef BTH_PGMPOOLKIND_PT_FOR_PT
114	# undef BTH_PGMPOOLKIND_ROOT
115	# undef PGM_BTH_NAME
116	# undef PGM_GST_TYPE
117	# undef PGM_GST_NAME
118
119	/* Guest - protected mode */
120	# define PGM_GST_TYPE PGM_TYPE_PROT
121	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
122	# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
123	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
124	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
125	# include "PGMGstDefs.h"
126	# include "PGMAllGst.h"
127	# include "PGMAllBth.h"
128	# undef BTH_PGMPOOLKIND_PT_FOR_PT
129	# undef BTH_PGMPOOLKIND_ROOT
130	# undef PGM_BTH_NAME
131	# undef PGM_GST_TYPE
132	# undef PGM_GST_NAME
133
134	/* Guest - 32-bit mode */
135	# define PGM_GST_TYPE PGM_TYPE_32BIT
136	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
137	# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
138	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
139	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
140	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
141	# include "PGMGstDefs.h"
142	# include "PGMAllGst.h"
143	# include "PGMAllBth.h"
144	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
145	# undef BTH_PGMPOOLKIND_PT_FOR_PT
146	# undef BTH_PGMPOOLKIND_ROOT
147	# undef PGM_BTH_NAME
148	# undef PGM_GST_TYPE
149	# undef PGM_GST_NAME
150
151	# undef PGM_SHW_TYPE
152	# undef PGM_SHW_NAME
153
154
155	/*
156	* Shadow - PAE mode
157	*/
158	# define PGM_SHW_TYPE PGM_TYPE_PAE
159	# define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
160	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
161	# include "PGMAllShw.h"
162
163	/* Guest - real mode */
164	# define PGM_GST_TYPE PGM_TYPE_REAL
165	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
166	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
167	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
168	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
169	# include "PGMGstDefs.h"
170	# include "PGMAllBth.h"
171	# undef BTH_PGMPOOLKIND_PT_FOR_PT
172	# undef BTH_PGMPOOLKIND_ROOT
173	# undef PGM_BTH_NAME
174	# undef PGM_GST_TYPE
175	# undef PGM_GST_NAME
176
177	/* Guest - protected mode */
178	# define PGM_GST_TYPE PGM_TYPE_PROT
179	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
180	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
181	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
182	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
183	# include "PGMGstDefs.h"
184	# include "PGMAllBth.h"
185	# undef BTH_PGMPOOLKIND_PT_FOR_PT
186	# undef BTH_PGMPOOLKIND_ROOT
187	# undef PGM_BTH_NAME
188	# undef PGM_GST_TYPE
189	# undef PGM_GST_NAME
190
191	/* Guest - 32-bit mode */
192	# define PGM_GST_TYPE PGM_TYPE_32BIT
193	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
194	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
195	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
196	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
197	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
198	# include "PGMGstDefs.h"
199	# include "PGMAllBth.h"
200	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
201	# undef BTH_PGMPOOLKIND_PT_FOR_PT
202	# undef BTH_PGMPOOLKIND_ROOT
203	# undef PGM_BTH_NAME
204	# undef PGM_GST_TYPE
205	# undef PGM_GST_NAME
206
207
208	/* Guest - PAE mode */
209	# define PGM_GST_TYPE PGM_TYPE_PAE
210	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
211	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
212	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
213	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
214	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
215	# include "PGMGstDefs.h"
216	# include "PGMAllGst.h"
217	# include "PGMAllBth.h"
218	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
219	# undef BTH_PGMPOOLKIND_PT_FOR_PT
220	# undef BTH_PGMPOOLKIND_ROOT
221	# undef PGM_BTH_NAME
222	# undef PGM_GST_TYPE
223	# undef PGM_GST_NAME
224
225	# undef PGM_SHW_TYPE
226	# undef PGM_SHW_NAME
227
228
229	/*
230	* Shadow - AMD64 mode
231	*/
232	# define PGM_SHW_TYPE PGM_TYPE_AMD64
233	# define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
234	# include "PGMAllShw.h"
235
236	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
237	/** @todo retire this hack. */
238	# define PGM_GST_TYPE PGM_TYPE_PROT
239	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
240	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
241	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
242	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
243	# include "PGMGstDefs.h"
244	# include "PGMAllBth.h"
245	# undef BTH_PGMPOOLKIND_PT_FOR_PT
246	# undef BTH_PGMPOOLKIND_ROOT
247	# undef PGM_BTH_NAME
248	# undef PGM_GST_TYPE
249	# undef PGM_GST_NAME
250
251	# ifdef VBOX_WITH_64_BITS_GUESTS
252	/* Guest - AMD64 mode */
253	# define PGM_GST_TYPE PGM_TYPE_AMD64
254	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
255	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
256	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
257	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
258	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
259	# include "PGMGstDefs.h"
260	# include "PGMAllGst.h"
261	# include "PGMAllBth.h"
262	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
263	# undef BTH_PGMPOOLKIND_PT_FOR_PT
264	# undef BTH_PGMPOOLKIND_ROOT
265	# undef PGM_BTH_NAME
266	# undef PGM_GST_TYPE
267	# undef PGM_GST_NAME
268	# endif /* VBOX_WITH_64_BITS_GUESTS */
269
270	# undef PGM_SHW_TYPE
271	# undef PGM_SHW_NAME
272
273
274	/*
275	* Shadow - 32-bit nested paging mode.
276	*/
277	# define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
278	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
279	# include "PGMAllShw.h"
280
281	/* Guest - real mode */
282	# define PGM_GST_TYPE PGM_TYPE_REAL
283	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
284	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
285	# include "PGMGstDefs.h"
286	# include "PGMAllBth.h"
287	# undef PGM_BTH_NAME
288	# undef PGM_GST_TYPE
289	# undef PGM_GST_NAME
290
291	/* Guest - protected mode */
292	# define PGM_GST_TYPE PGM_TYPE_PROT
293	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
294	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
295	# include "PGMGstDefs.h"
296	# include "PGMAllBth.h"
297	# undef PGM_BTH_NAME
298	# undef PGM_GST_TYPE
299	# undef PGM_GST_NAME
300
301	/* Guest - 32-bit mode */
302	# define PGM_GST_TYPE PGM_TYPE_32BIT
303	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
304	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
305	# include "PGMGstDefs.h"
306	# include "PGMAllBth.h"
307	# undef PGM_BTH_NAME
308	# undef PGM_GST_TYPE
309	# undef PGM_GST_NAME
310
311	/* Guest - PAE mode */
312	# define PGM_GST_TYPE PGM_TYPE_PAE
313	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
314	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
315	# include "PGMGstDefs.h"
316	# include "PGMAllBth.h"
317	# undef PGM_BTH_NAME
318	# undef PGM_GST_TYPE
319	# undef PGM_GST_NAME
320
321	# ifdef VBOX_WITH_64_BITS_GUESTS
322	/* Guest - AMD64 mode */
323	# define PGM_GST_TYPE PGM_TYPE_AMD64
324	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
325	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
326	# include "PGMGstDefs.h"
327	# include "PGMAllBth.h"
328	# undef PGM_BTH_NAME
329	# undef PGM_GST_TYPE
330	# undef PGM_GST_NAME
331	# endif /* VBOX_WITH_64_BITS_GUESTS */
332
333	# undef PGM_SHW_TYPE
334	# undef PGM_SHW_NAME
335
336
337	/*
338	* Shadow - PAE nested paging mode.
339	*/
340	# define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
341	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
342	# include "PGMAllShw.h"
343
344	/* Guest - real mode */
345	# define PGM_GST_TYPE PGM_TYPE_REAL
346	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
347	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
348	# include "PGMGstDefs.h"
349	# include "PGMAllBth.h"
350	# undef PGM_BTH_NAME
351	# undef PGM_GST_TYPE
352	# undef PGM_GST_NAME
353
354	/* Guest - protected mode */
355	# define PGM_GST_TYPE PGM_TYPE_PROT
356	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
357	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
358	# include "PGMGstDefs.h"
359	# include "PGMAllBth.h"
360	# undef PGM_BTH_NAME
361	# undef PGM_GST_TYPE
362	# undef PGM_GST_NAME
363
364	/* Guest - 32-bit mode */
365	# define PGM_GST_TYPE PGM_TYPE_32BIT
366	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
367	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
368	# include "PGMGstDefs.h"
369	# include "PGMAllBth.h"
370	# undef PGM_BTH_NAME
371	# undef PGM_GST_TYPE
372	# undef PGM_GST_NAME
373
374	/* Guest - PAE mode */
375	# define PGM_GST_TYPE PGM_TYPE_PAE
376	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
377	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
378	# include "PGMGstDefs.h"
379	# include "PGMAllBth.h"
380	# undef PGM_BTH_NAME
381	# undef PGM_GST_TYPE
382	# undef PGM_GST_NAME
383
384	# ifdef VBOX_WITH_64_BITS_GUESTS
385	/* Guest - AMD64 mode */
386	# define PGM_GST_TYPE PGM_TYPE_AMD64
387	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
388	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
389	# include "PGMGstDefs.h"
390	# include "PGMAllBth.h"
391	# undef PGM_BTH_NAME
392	# undef PGM_GST_TYPE
393	# undef PGM_GST_NAME
394	# endif /* VBOX_WITH_64_BITS_GUESTS */
395
396	# undef PGM_SHW_TYPE
397	# undef PGM_SHW_NAME
398
399
400	/*
401	* Shadow - AMD64 nested paging mode.
402	*/
403	# define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
404	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
405	# include "PGMAllShw.h"
406
407	/* Guest - real mode */
408	# define PGM_GST_TYPE PGM_TYPE_REAL
409	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
410	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
411	# include "PGMGstDefs.h"
412	# include "PGMAllBth.h"
413	# undef PGM_BTH_NAME
414	# undef PGM_GST_TYPE
415	# undef PGM_GST_NAME
416
417	/* Guest - protected mode */
418	# define PGM_GST_TYPE PGM_TYPE_PROT
419	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
420	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
421	# include "PGMGstDefs.h"
422	# include "PGMAllBth.h"
423	# undef PGM_BTH_NAME
424	# undef PGM_GST_TYPE
425	# undef PGM_GST_NAME
426
427	/* Guest - 32-bit mode */
428	# define PGM_GST_TYPE PGM_TYPE_32BIT
429	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
430	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
431	# include "PGMGstDefs.h"
432	# include "PGMAllBth.h"
433	# undef PGM_BTH_NAME
434	# undef PGM_GST_TYPE
435	# undef PGM_GST_NAME
436
437	/* Guest - PAE mode */
438	# define PGM_GST_TYPE PGM_TYPE_PAE
439	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
440	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
441	# include "PGMGstDefs.h"
442	# include "PGMAllBth.h"
443	# undef PGM_BTH_NAME
444	# undef PGM_GST_TYPE
445	# undef PGM_GST_NAME
446
447	# ifdef VBOX_WITH_64_BITS_GUESTS
448	/* Guest - AMD64 mode */
449	# define PGM_GST_TYPE PGM_TYPE_AMD64
450	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
451	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
452	# include "PGMGstDefs.h"
453	# include "PGMAllBth.h"
454	# undef PGM_BTH_NAME
455	# undef PGM_GST_TYPE
456	# undef PGM_GST_NAME
457	# endif /* VBOX_WITH_64_BITS_GUESTS */
458
459	# undef PGM_SHW_TYPE
460	# undef PGM_SHW_NAME
461
462
463	/*
464	* Shadow - EPT.
465	*/
466	# define PGM_SHW_TYPE PGM_TYPE_EPT
467	# define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
468	# include "PGMAllShw.h"
469
470	/* Guest - real mode */
471	# define PGM_GST_TYPE PGM_TYPE_REAL
472	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
473	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
474	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
475	# include "PGMGstDefs.h"
476	# include "PGMAllBth.h"
477	# undef BTH_PGMPOOLKIND_PT_FOR_PT
478	# undef PGM_BTH_NAME
479	# undef PGM_GST_TYPE
480	# undef PGM_GST_NAME
481
482	/* Guest - protected mode */
483	# define PGM_GST_TYPE PGM_TYPE_PROT
484	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
485	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
486	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
487	# include "PGMGstDefs.h"
488	# include "PGMAllBth.h"
489	# undef BTH_PGMPOOLKIND_PT_FOR_PT
490	# undef PGM_BTH_NAME
491	# undef PGM_GST_TYPE
492	# undef PGM_GST_NAME
493
494	/* Guest - 32-bit mode */
495	# define PGM_GST_TYPE PGM_TYPE_32BIT
496	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
497	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
498	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
499	# include "PGMGstDefs.h"
500	# include "PGMAllBth.h"
501	# undef BTH_PGMPOOLKIND_PT_FOR_PT
502	# undef PGM_BTH_NAME
503	# undef PGM_GST_TYPE
504	# undef PGM_GST_NAME
505
506	/* Guest - PAE mode */
507	# define PGM_GST_TYPE PGM_TYPE_PAE
508	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
509	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
510	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
511	# include "PGMGstDefs.h"
512	# include "PGMAllBth.h"
513	# undef BTH_PGMPOOLKIND_PT_FOR_PT
514	# undef PGM_BTH_NAME
515	# undef PGM_GST_TYPE
516	# undef PGM_GST_NAME
517
518	# ifdef VBOX_WITH_64_BITS_GUESTS
519	/* Guest - AMD64 mode */
520	# define PGM_GST_TYPE PGM_TYPE_AMD64
521	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
522	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
523	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
524	# include "PGMGstDefs.h"
525	# include "PGMAllBth.h"
526	# undef BTH_PGMPOOLKIND_PT_FOR_PT
527	# undef PGM_BTH_NAME
528	# undef PGM_GST_TYPE
529	# undef PGM_GST_NAME
530	# endif /* VBOX_WITH_64_BITS_GUESTS */
531
532	# undef PGM_SHW_TYPE
533	# undef PGM_SHW_NAME
534
535
536	/*
537	* Shadow - NEM / None.
538	*/
539	# define PGM_SHW_TYPE PGM_TYPE_NONE
540	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
541	# include "PGMAllShw.h"
542
543	/* Guest - real mode */
544	# define PGM_GST_TYPE PGM_TYPE_REAL
545	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
546	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
547	# include "PGMGstDefs.h"
548	# include "PGMAllBth.h"
549	# undef PGM_BTH_NAME
550	# undef PGM_GST_TYPE
551	# undef PGM_GST_NAME
552
553	/* Guest - protected mode */
554	# define PGM_GST_TYPE PGM_TYPE_PROT
555	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
556	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
557	# include "PGMGstDefs.h"
558	# include "PGMAllBth.h"
559	# undef PGM_BTH_NAME
560	# undef PGM_GST_TYPE
561	# undef PGM_GST_NAME
562
563	/* Guest - 32-bit mode */
564	# define PGM_GST_TYPE PGM_TYPE_32BIT
565	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
566	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
567	# include "PGMGstDefs.h"
568	# include "PGMAllBth.h"
569	# undef PGM_BTH_NAME
570	# undef PGM_GST_TYPE
571	# undef PGM_GST_NAME
572
573	/* Guest - PAE mode */
574	# define PGM_GST_TYPE PGM_TYPE_PAE
575	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
576	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
577	# include "PGMGstDefs.h"
578	# include "PGMAllBth.h"
579	# undef PGM_BTH_NAME
580	# undef PGM_GST_TYPE
581	# undef PGM_GST_NAME
582
583	# ifdef VBOX_WITH_64_BITS_GUESTS
584	/* Guest - AMD64 mode */
585	# define PGM_GST_TYPE PGM_TYPE_AMD64
586	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
587	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
588	# include "PGMGstDefs.h"
589	# include "PGMAllBth.h"
590	# undef PGM_BTH_NAME
591	# undef PGM_GST_TYPE
592	# undef PGM_GST_NAME
593	# endif /* VBOX_WITH_64_BITS_GUESTS */
594
595	# undef PGM_SHW_TYPE
596	# undef PGM_SHW_NAME
597
598
599
600	/**
601	* Guest mode data array.
602	*/
603	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
604	{
605	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
606	{
607	PGM_TYPE_REAL,
608	PGM_GST_NAME_REAL(GetPage),
609	PGM_GST_NAME_REAL(QueryPageFast),
610	PGM_GST_NAME_REAL(ModifyPage),
611	PGM_GST_NAME_REAL(Enter),
612	PGM_GST_NAME_REAL(Exit),
613	# ifdef IN_RING3
614	PGM_GST_NAME_REAL(Relocate),
615	# endif
616	},
617	{
618	PGM_TYPE_PROT,
619	PGM_GST_NAME_PROT(GetPage),
620	PGM_GST_NAME_PROT(QueryPageFast),
621	PGM_GST_NAME_PROT(ModifyPage),
622	PGM_GST_NAME_PROT(Enter),
623	PGM_GST_NAME_PROT(Exit),
624	# ifdef IN_RING3
625	PGM_GST_NAME_PROT(Relocate),
626	# endif
627	},
628	{
629	PGM_TYPE_32BIT,
630	PGM_GST_NAME_32BIT(GetPage),
631	PGM_GST_NAME_32BIT(QueryPageFast),
632	PGM_GST_NAME_32BIT(ModifyPage),
633	PGM_GST_NAME_32BIT(Enter),
634	PGM_GST_NAME_32BIT(Exit),
635	# ifdef IN_RING3
636	PGM_GST_NAME_32BIT(Relocate),
637	# endif
638	},
639	{
640	PGM_TYPE_PAE,
641	PGM_GST_NAME_PAE(GetPage),
642	PGM_GST_NAME_PAE(QueryPageFast),
643	PGM_GST_NAME_PAE(ModifyPage),
644	PGM_GST_NAME_PAE(Enter),
645	PGM_GST_NAME_PAE(Exit),
646	# ifdef IN_RING3
647	PGM_GST_NAME_PAE(Relocate),
648	# endif
649	},
650	# ifdef VBOX_WITH_64_BITS_GUESTS
651	{
652	PGM_TYPE_AMD64,
653	PGM_GST_NAME_AMD64(GetPage),
654	PGM_GST_NAME_AMD64(QueryPageFast),
655	PGM_GST_NAME_AMD64(ModifyPage),
656	PGM_GST_NAME_AMD64(Enter),
657	PGM_GST_NAME_AMD64(Exit),
658	# ifdef IN_RING3
659	PGM_GST_NAME_AMD64(Relocate),
660	# endif
661	},
662	# endif
663	};
664
665
666	/**
667	* The shadow mode data array.
668	*/
669	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
670	{
671	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
672	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
673	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
674	{
675	PGM_TYPE_32BIT,
676	PGM_SHW_NAME_32BIT(GetPage),
677	PGM_SHW_NAME_32BIT(ModifyPage),
678	PGM_SHW_NAME_32BIT(Enter),
679	PGM_SHW_NAME_32BIT(Exit),
680	# ifdef IN_RING3
681	PGM_SHW_NAME_32BIT(Relocate),
682	# endif
683	},
684	{
685	PGM_TYPE_PAE,
686	PGM_SHW_NAME_PAE(GetPage),
687	PGM_SHW_NAME_PAE(ModifyPage),
688	PGM_SHW_NAME_PAE(Enter),
689	PGM_SHW_NAME_PAE(Exit),
690	# ifdef IN_RING3
691	PGM_SHW_NAME_PAE(Relocate),
692	# endif
693	},
694	{
695	PGM_TYPE_AMD64,
696	PGM_SHW_NAME_AMD64(GetPage),
697	PGM_SHW_NAME_AMD64(ModifyPage),
698	PGM_SHW_NAME_AMD64(Enter),
699	PGM_SHW_NAME_AMD64(Exit),
700	# ifdef IN_RING3
701	PGM_SHW_NAME_AMD64(Relocate),
702	# endif
703	},
704	{
705	PGM_TYPE_NESTED_32BIT,
706	PGM_SHW_NAME_NESTED_32BIT(GetPage),
707	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
708	PGM_SHW_NAME_NESTED_32BIT(Enter),
709	PGM_SHW_NAME_NESTED_32BIT(Exit),
710	# ifdef IN_RING3
711	PGM_SHW_NAME_NESTED_32BIT(Relocate),
712	# endif
713	},
714	{
715	PGM_TYPE_NESTED_PAE,
716	PGM_SHW_NAME_NESTED_PAE(GetPage),
717	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
718	PGM_SHW_NAME_NESTED_PAE(Enter),
719	PGM_SHW_NAME_NESTED_PAE(Exit),
720	# ifdef IN_RING3
721	PGM_SHW_NAME_NESTED_PAE(Relocate),
722	# endif
723	},
724	{
725	PGM_TYPE_NESTED_AMD64,
726	PGM_SHW_NAME_NESTED_AMD64(GetPage),
727	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
728	PGM_SHW_NAME_NESTED_AMD64(Enter),
729	PGM_SHW_NAME_NESTED_AMD64(Exit),
730	# ifdef IN_RING3
731	PGM_SHW_NAME_NESTED_AMD64(Relocate),
732	# endif
733	},
734	{
735	PGM_TYPE_EPT,
736	PGM_SHW_NAME_EPT(GetPage),
737	PGM_SHW_NAME_EPT(ModifyPage),
738	PGM_SHW_NAME_EPT(Enter),
739	PGM_SHW_NAME_EPT(Exit),
740	# ifdef IN_RING3
741	PGM_SHW_NAME_EPT(Relocate),
742	# endif
743	},
744	{
745	PGM_TYPE_NONE,
746	PGM_SHW_NAME_NONE(GetPage),
747	PGM_SHW_NAME_NONE(ModifyPage),
748	PGM_SHW_NAME_NONE(Enter),
749	PGM_SHW_NAME_NONE(Exit),
750	# ifdef IN_RING3
751	PGM_SHW_NAME_NONE(Relocate),
752	# endif
753	},
754	};
755
756
757	/**
758	* The guest+shadow mode data array.
759	*/
760	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
761	{
762	# if !defined(IN_RING3) && !defined(VBOX_STRICT)
763	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
764	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
765	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
766
767	# elif !defined(IN_RING3) && defined(VBOX_STRICT)
768	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
769	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
770	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
771
772	# elif defined(IN_RING3) && !defined(VBOX_STRICT)
773	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
774	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
775	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
776
777	# elif defined(IN_RING3) && defined(VBOX_STRICT)
778	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
779	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
780	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
781
782	# else
783	# error "Misconfig."
784	# endif
785
786	/* 32-bit shadow paging mode: */
787	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
788	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
789	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
790	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
791	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
793	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
796	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
797	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
798
799	/* PAE shadow paging mode: */
800	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
801	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
802	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
803	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
805	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
806	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
807	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
808	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
810	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
811
812	/* AMD64 shadow paging mode: */
813	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
814	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
815	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
816	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
817	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
818	# ifdef VBOX_WITH_64_BITS_GUESTS
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
820	# else
821	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
822	# endif
823	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
824	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
825	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
827	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
828
829	/* 32-bit nested paging mode: */
830	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
831	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
832	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
833	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
834	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
835	# ifdef VBOX_WITH_64_BITS_GUESTS
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
837	# else
838	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
839	# endif
840	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
841	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
842	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
844	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
845
846	/* PAE nested paging mode: */
847	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
848	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
849	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
850	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
851	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
852	# ifdef VBOX_WITH_64_BITS_GUESTS
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
854	# else
855	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
856	# endif
857	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
858	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
859	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
861	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
862
863	/* AMD64 nested paging mode: */
864	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
865	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
866	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
867	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
868	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
869	# ifdef VBOX_WITH_64_BITS_GUESTS
870	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
871	# else
872	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
873	# endif
874	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
875	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
876	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
877	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
878	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
879
880	/* EPT nested paging mode: */
881	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
882	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
883	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
884	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
885	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
886	# ifdef VBOX_WITH_64_BITS_GUESTS
887	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
888	# else
889	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
890	# endif
891	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
892	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
893	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
894	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
895	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
896
897	/* NONE / NEM: */
898	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
899	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
900	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
901	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
902	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
903	# ifdef VBOX_WITH_64_BITS_GUESTS
904	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
905	# else
906	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
907	# endif
908	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
909	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
910	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
911	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
912	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
913
914
915	# undef PGMMODEDATABTH_ENTRY
916	# undef PGMMODEDATABTH_NULL_ENTRY
917	};
918
919
920	/** Mask array used by pgmGetCr3MaskForMode.
921	* X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
922	static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
923	{
924	/* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK,
925	/* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK,
926	/* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK,
927	/* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK,
928	/* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK,
929	/* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK,
930	/* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
931	/* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK,
932	/* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK,
933	/* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK,
934	/* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
935	/* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK,
936	/* [12] = */ 0,
937	/* [13] = */ 0,
938	/* [14] = */ 0,
939	/* [15] = */ 0,
940	/* [16] = */ 0,
941	/* [17] = */ 0,
942	/* [18] = */ 0,
943	/* [19] = */ 0,
944	/* [20] = */ 0,
945	/* [21] = */ 0,
946	/* [22] = */ 0,
947	/* [23] = */ 0,
948	/* [24] = */ 0,
949	/* [25] = */ 0,
950	/* [26] = */ 0,
951	/* [27] = */ 0,
952	/* [28] = */ 0,
953	/* [29] = */ 0,
954	/* [30] = */ 0,
955	/* [31] = */ 0,
956	/* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK,
957	};
958	AssertCompile(PGMMODE_NONE == 32);
959
960
961	#elif defined(VBOX_VMM_TARGET_ARMV8)
962
963	/** @todo Proper ARMv8 guest support for PGM */
964	# include "PGMAllGst-armv8.h"
965
966	#else
967	# error "port me"
968	#endif
969
970
971	#ifdef VBOX_VMM_TARGET_X86
972
973	/**
974	* Gets the physical address mask for CR3 in the given paging mode.
975	*
976	* The mask is for eliminating flags and other stuff in CR3/EPTP when
977	* extracting the physical address. It is not for validating whether there are
978	* reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed
979	* it to PGM checked for reserved bits, including reserved physical address
980	* bits.
981	*
982	* @returns The CR3 mask.
983	* @param enmMode The paging mode.
984	* @param enmSlatMode The second-level address translation mode.
985	*/
986	DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
987	{
988	if (enmSlatMode == PGMSLAT_DIRECT)
989	{
990	Assert(enmMode != PGMMODE_EPT);
991	return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
992	}
993	Assert(enmSlatMode == PGMSLAT_EPT);
994	return X86_CR3_EPT_PAGE_MASK;
995	}
996
997
998	/**
999	* Gets the masked CR3 value according to the current guest paging mode.
1000	*
1001	* See disclaimer in pgmGetCr3MaskForMode.
1002	*
1003	* @returns The masked PGM CR3 value.
1004	* @param pVCpu The cross context virtual CPU structure.
1005	* @param uCr3 The raw guest CR3 value.
1006	*/
1007	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
1008	{
1009	uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
1010	RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
1011	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
1012	return GCPhysCR3;
1013	}
1014
1015
1016	# ifdef IN_RING0
1017	/**
1018	* \#PF Handler.
1019	*
1020	* @returns VBox status code (appropriate for trap handling and GC return).
1021	* @param pVCpu The cross context virtual CPU structure.
1022	* @param uErr The trap error code.
1023	* @param pCtx Pointer to the register context for the CPU.
1024	* @param pvFault The fault address.
1025	*/
1026	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
1027	{
1028	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1029
1030	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
1031	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
1032	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
1033
1034
1035	# ifdef VBOX_WITH_STATISTICS
1036	/*
1037	* Error code stats.
1038	*/
1039	if (uErr & X86_TRAP_PF_US)
1040	{
1041	if (!(uErr & X86_TRAP_PF_P))
1042	{
1043	if (uErr & X86_TRAP_PF_RW)
1044	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
1045	else
1046	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
1047	}
1048	else if (uErr & X86_TRAP_PF_RW)
1049	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1050	else if (uErr & X86_TRAP_PF_RSVD)
1051	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1052	else if (uErr & X86_TRAP_PF_ID)
1053	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1054	else
1055	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1056	}
1057	else
1058	{ /* Supervisor */
1059	if (!(uErr & X86_TRAP_PF_P))
1060	{
1061	if (uErr & X86_TRAP_PF_RW)
1062	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1063	else
1064	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1065	}
1066	else if (uErr & X86_TRAP_PF_RW)
1067	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1068	else if (uErr & X86_TRAP_PF_ID)
1069	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1070	else if (uErr & X86_TRAP_PF_RSVD)
1071	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1072	}
1073	# endif /* VBOX_WITH_STATISTICS */
1074
1075	/*
1076	* Call the worker.
1077	*/
1078	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1079	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1080	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
1081	bool fLockTaken = false;
1082	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
1083	if (fLockTaken)
1084	{
1085	PGM_LOCK_ASSERT_OWNER(pVM);
1086	PGM_UNLOCK(pVM);
1087	}
1088	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
1089
1090	/*
1091	* Return code tweaks.
1092	*/
1093	if (rc != VINF_SUCCESS)
1094	{
1095	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1096	rc = VINF_SUCCESS;
1097
1098	/* Note: hack alert for difficult to reproduce problem. */
1099	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1100	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1101	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1102	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1103	{
1104	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
1105	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
1106	rc = VINF_SUCCESS;
1107	}
1108	}
1109
1110	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
1111	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1112	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1113	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1114	return rc;
1115	}
1116	# endif /* IN_RING0 */
1117
1118
1119	/**
1120	* Prefetch a page
1121	*
1122	* Typically used to sync commonly used pages before entering raw mode
1123	* after a CR3 reload.
1124	*
1125	* @returns VBox status code suitable for scheduling.
1126	* @retval VINF_SUCCESS on success.
1127	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1128	* @param pVCpu The cross context virtual CPU structure.
1129	* @param GCPtrPage Page to invalidate.
1130	*/
1131	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1132	{
1133	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1134
1135	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1136	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1137	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1138	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1139
1140	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1141	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1142	return rc;
1143	}
1144
1145
1146	/**
1147	* Emulation of the invlpg instruction (HC only actually).
1148	*
1149	* @returns Strict VBox status code, special care required.
1150	* @retval VINF_PGM_SYNC_CR3 - handled.
1151	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1152	*
1153	* @param pVCpu The cross context virtual CPU structure.
1154	* @param GCPtrPage Page to invalidate.
1155	*
1156	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1157	* safe, but there could be edge cases!
1158	*
1159	* @todo Flush page or page directory only if necessary!
1160	* @todo VBOXSTRICTRC
1161	*/
1162	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1163	{
1164	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1165	int rc;
1166	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1167
1168	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1169
1170	/*
1171	* Call paging mode specific worker.
1172	*/
1173	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1174	PGM_LOCK_VOID(pVM);
1175
1176	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1177	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1178	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1179	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1180
1181	PGM_UNLOCK(pVM);
1182	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1183
1184	/* Ignore all irrelevant error codes. */
1185	if ( rc == VERR_PAGE_NOT_PRESENT
1186	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1187	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1188	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1189	rc = VINF_SUCCESS;
1190
1191	return rc;
1192	}
1193
1194
1195	/**
1196	* Executes an instruction using the interpreter.
1197	*
1198	* @returns VBox status code (appropriate for trap handling and GC return).
1199	* @param pVCpu The cross context virtual CPU structure.
1200	* @param pvFault Fault address.
1201	*/
1202	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
1203	{
1204	RT_NOREF(pvFault);
1205	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
1206	if (rc == VERR_EM_INTERPRETER)
1207	rc = VINF_EM_RAW_EMULATE_INSTR;
1208	if (rc != VINF_SUCCESS)
1209	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1210	return rc;
1211	}
1212
1213
1214	/**
1215	* Gets effective page information (from the VMM page directory).
1216	*
1217	* @returns VBox status code.
1218	* @param pVCpu The cross context virtual CPU structure.
1219	* @param GCPtr Guest Context virtual address of the page.
1220	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1221	* @param pHCPhys Where to store the HC physical address of the page.
1222	* This is page aligned.
1223	* @remark You should use PGMMapGetPage() for pages in a mapping.
1224	*/
1225	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1226	{
1227	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1228	PGM_LOCK_VOID(pVM);
1229
1230	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1231	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1232	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1233	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1234
1235	PGM_UNLOCK(pVM);
1236	return rc;
1237	}
1238
1239
1240	/**
1241	* Modify page flags for a range of pages in the shadow context.
1242	*
1243	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1244	*
1245	* @returns VBox status code.
1246	* @param pVCpu The cross context virtual CPU structure.
1247	* @param GCPtr Virtual address of the first page in the range.
1248	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1249	* @param fMask The AND mask - page flags X86_PTE_*.
1250	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1251	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1252	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1253	*/
1254	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1255	{
1256	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1257	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1258
1259	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1260
1261	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1262	PGM_LOCK_VOID(pVM);
1263
1264	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1265	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1266	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1267	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1268
1269	PGM_UNLOCK(pVM);
1270	return rc;
1271	}
1272
1273
1274	/**
1275	* Changing the page flags for a single page in the shadow page tables so as to
1276	* make it read-only.
1277	*
1278	* @returns VBox status code.
1279	* @param pVCpu The cross context virtual CPU structure.
1280	* @param GCPtr Virtual address of the first page in the range.
1281	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1282	*/
1283	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1284	{
1285	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1286	}
1287
1288
1289	/**
1290	* Changing the page flags for a single page in the shadow page tables so as to
1291	* make it writable.
1292	*
1293	* The call must know with 101% certainty that the guest page tables maps this
1294	* as writable too. This function will deal shared, zero and write monitored
1295	* pages.
1296	*
1297	* @returns VBox status code.
1298	* @param pVCpu The cross context virtual CPU structure.
1299	* @param GCPtr Virtual address of the first page in the range.
1300	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1301	*/
1302	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1303	{
1304	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1305	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1306	return VINF_SUCCESS;
1307	}
1308
1309
1310	/**
1311	* Changing the page flags for a single page in the shadow page tables so as to
1312	* make it not present.
1313	*
1314	* @returns VBox status code.
1315	* @param pVCpu The cross context virtual CPU structure.
1316	* @param GCPtr Virtual address of the first page in the range.
1317	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1318	*/
1319	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1320	{
1321	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1322	}
1323
1324	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
1325
1326	/**
1327	* Changing the page flags for a single page in the shadow page tables so as to
1328	* make it supervisor and writable.
1329	*
1330	* This if for dealing with CR0.WP=0 and readonly user pages.
1331	*
1332	* @returns VBox status code.
1333	* @param pVCpu The cross context virtual CPU structure.
1334	* @param GCPtr Virtual address of the first page in the range.
1335	* @param fBigPage Whether or not this is a big page. If it is, we have to
1336	* change the shadow PDE as well. If it isn't, the caller
1337	* has checked that the shadow PDE doesn't need changing.
1338	* We ASSUME 4KB pages backing the big page here!
1339	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1340	*/
1341	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1342	{
1343	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1344	if (rc == VINF_SUCCESS && fBigPage)
1345	{
1346	/* this is a bit ugly... */
1347	switch (pVCpu->pgm.s.enmShadowMode)
1348	{
1349	case PGMMODE_32_BIT:
1350	{
1351	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1352	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1353	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1354	pPde->u \|= X86_PDE_RW;
1355	Log(("-> PDE=%#llx (32)\n", pPde->u));
1356	break;
1357	}
1358	case PGMMODE_PAE:
1359	case PGMMODE_PAE_NX:
1360	{
1361	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1362	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1363	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1364	pPde->u \|= X86_PDE_RW;
1365	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1366	break;
1367	}
1368	default:
1369	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1370	}
1371	}
1372	return rc;
1373	}
1374
1375
1376	/**
1377	* Gets the shadow page directory for the specified address, PAE.
1378	*
1379	* @returns Pointer to the shadow PD.
1380	* @param pVCpu The cross context virtual CPU structure.
1381	* @param GCPtr The address.
1382	* @param uGstPdpe Guest PDPT entry. Valid.
1383	* @param ppPD Receives address of page directory
1384	*/
1385	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1386	{
1387	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1388	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1389	PPGMPOOLPAGE pShwPage;
1390	int rc;
1391	PGM_LOCK_ASSERT_OWNER(pVM);
1392
1393
1394	/* Allocate page directory if not present. */
1395	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1396	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1397	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1398	X86PGPAEUINT const uPdpe = pPdpe->u;
1399	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1400	{
1401	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1402	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1403	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1404
1405	pgmPoolCacheUsed(pPool, pShwPage);
1406
1407	/* Update the entry if necessary. */
1408	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1409	if (uPdpeNew == uPdpe)
1410	{ /* likely */ }
1411	else
1412	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1413	}
1414	else
1415	{
1416	RTGCPTR64 GCPdPt;
1417	PGMPOOLKIND enmKind;
1418	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1419	{
1420	/* AMD-V nested paging or real/protected mode without paging. */
1421	GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1422	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1423	}
1424	else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1425	{
1426	if (uGstPdpe & X86_PDPE_P)
1427	{
1428	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1429	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1430	}
1431	else
1432	{
1433	/* PD not present; guest must reload CR3 to change it.
1434	* No need to monitor anything in this case. */
1435	/** @todo r=bird: WTF is hit?!? */
1436	/Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? /
1437	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1438	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1439	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1440	}
1441	}
1442	else
1443	{
1444	GCPdPt = CPUMGetGuestCR3(pVCpu);
1445	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1446	}
1447
1448	/* Create a reference back to the PDPT by using the index in its shadow page. */
1449	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1450	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1451	&pShwPage);
1452	AssertRCReturn(rc, rc);
1453
1454	/* Hook it up. */
1455	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1456	}
1457	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1458
1459	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1460	return VINF_SUCCESS;
1461	}
1462
1463
1464	/**
1465	* Gets the pointer to the shadow page directory entry for an address, PAE.
1466	*
1467	* @returns Pointer to the PDE.
1468	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1469	* @param GCPtr The address.
1470	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1471	*/
1472	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1473	{
1474	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1475	PGM_LOCK_ASSERT_OWNER(pVM);
1476
1477	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1478	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1479	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1480	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1481	if (!(uPdpe & X86_PDPE_P))
1482	{
1483	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1484	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1485	}
1486	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1487
1488	/* Fetch the pgm pool shadow descriptor. */
1489	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1490	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1491
1492	*ppShwPde = pShwPde;
1493	return VINF_SUCCESS;
1494	}
1495
1496
1497	/**
1498	* Syncs the SHADOW page directory pointer for the specified address.
1499	*
1500	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1501	*
1502	* The caller is responsible for making sure the guest has a valid PD before
1503	* calling this function.
1504	*
1505	* @returns VBox status code.
1506	* @param pVCpu The cross context virtual CPU structure.
1507	* @param GCPtr The address.
1508	* @param uGstPml4e Guest PML4 entry (valid).
1509	* @param uGstPdpe Guest PDPT entry (valid).
1510	* @param ppPD Receives address of page directory
1511	*/
1512	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1513	{
1514	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1515	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1516	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1517	int rc;
1518
1519	PGM_LOCK_ASSERT_OWNER(pVM);
1520
1521	/*
1522	* PML4.
1523	*/
1524	PPGMPOOLPAGE pShwPage;
1525	{
1526	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1527	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1528	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1529	X86PGPAEUINT const uPml4e = pPml4e->u;
1530
1531	/* Allocate page directory pointer table if not present. */
1532	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1533	{
1534	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1535	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1536
1537	pgmPoolCacheUsed(pPool, pShwPage);
1538
1539	/* Update the entry if needed. */
1540	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1541	\| (uPml4e & PGM_PML4_FLAGS);
1542	if (uPml4e == uPml4eNew)
1543	{ /* likely */ }
1544	else
1545	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1546	}
1547	else
1548	{
1549	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1550
1551	RTGCPTR64 GCPml4;
1552	PGMPOOLKIND enmKind;
1553	if (fNestedPagingOrNoGstPaging)
1554	{
1555	/* AMD-V nested paging or real/protected mode without paging */
1556	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1557	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1558	}
1559	else
1560	{
1561	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1562	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1563	}
1564
1565	/* Create a reference back to the PDPT by using the index in its shadow page. */
1566	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1567	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1568	&pShwPage);
1569	AssertRCReturn(rc, rc);
1570
1571	/* Hook it up. */
1572	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1573	\| (uPml4e & PGM_PML4_FLAGS));
1574	}
1575	}
1576
1577	/*
1578	* PDPT.
1579	*/
1580	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1581	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1582	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1583	X86PGPAEUINT const uPdpe = pPdpe->u;
1584
1585	/* Allocate page directory if not present. */
1586	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1587	{
1588	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1589	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1590
1591	pgmPoolCacheUsed(pPool, pShwPage);
1592
1593	/* Update the entry if needed. */
1594	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1595	\| (uPdpe & PGM_PDPT_FLAGS);
1596	if (uPdpe == uPdpeNew)
1597	{ /* likely */ }
1598	else
1599	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1600	}
1601	else
1602	{
1603	RTGCPTR64 GCPdPt;
1604	PGMPOOLKIND enmKind;
1605	if (fNestedPagingOrNoGstPaging)
1606	{
1607	/* AMD-V nested paging or real/protected mode without paging */
1608	GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1609	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1610	}
1611	else
1612	{
1613	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1614	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1615	}
1616
1617	/* Create a reference back to the PDPT by using the index in its shadow page. */
1618	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1619	pShwPage->idx, iPdPt, false /fLockPage/,
1620	&pShwPage);
1621	AssertRCReturn(rc, rc);
1622
1623	/* Hook it up. */
1624	ASMAtomicWriteU64(&pPdpe->u,
1625	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1626	}
1627
1628	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1629	return VINF_SUCCESS;
1630	}
1631
1632
1633	/**
1634	* Gets the SHADOW page directory pointer for the specified address (long mode).
1635	*
1636	* @returns VBox status code.
1637	* @param pVCpu The cross context virtual CPU structure.
1638	* @param GCPtr The address.
1639	* @param ppPml4e Receives the address of the page map level 4 entry.
1640	* @param ppPdpt Receives the address of the page directory pointer table.
1641	* @param ppPD Receives the address of the page directory.
1642	*/
1643	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1644	{
1645	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1646	PGM_LOCK_ASSERT_OWNER(pVM);
1647
1648	/*
1649	* PML4
1650	*/
1651	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1652	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1653	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1654	if (ppPml4e)
1655	*ppPml4e = (PX86PML4E)pPml4e;
1656	X86PGPAEUINT const uPml4e = pPml4e->u;
1657	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1658	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1659	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1660
1661	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1662	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1663	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1664
1665	/*
1666	* PDPT
1667	*/
1668	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1669	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1670	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1671	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1672	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1673
1674	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1675	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1676
1677	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1678	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1679	return VINF_SUCCESS;
1680	}
1681
1682
1683	/**
1684	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1685	* backing pages in case the PDPT or PML4 entry is missing.
1686	*
1687	* @returns VBox status code.
1688	* @param pVCpu The cross context virtual CPU structure.
1689	* @param GCPtr The address.
1690	* @param ppPdpt Receives address of pdpt
1691	* @param ppPD Receives address of page directory
1692	*/
1693	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1694	{
1695	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1696	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1697	int rc;
1698
1699	Assert(pVM->pgm.s.fNestedPaging);
1700	PGM_LOCK_ASSERT_OWNER(pVM);
1701
1702	/*
1703	* PML4 level.
1704	*/
1705	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1706	Assert(pPml4);
1707
1708	/* Allocate page directory pointer table if not present. */
1709	PPGMPOOLPAGE pShwPage;
1710	{
1711	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1712	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1713	EPTPML4E Pml4e;
1714	Pml4e.u = pPml4e->u;
1715	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1716	{
1717	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1718	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1719	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1720	&pShwPage);
1721	AssertRCReturn(rc, rc);
1722
1723	/* Hook up the new PDPT now. */
1724	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1725	}
1726	else
1727	{
1728	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1729	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1730
1731	pgmPoolCacheUsed(pPool, pShwPage);
1732
1733	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1734	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1735	{ }
1736	else
1737	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1738	}
1739	}
1740
1741	/*
1742	* PDPT level.
1743	*/
1744	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1745	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1746	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1747
1748	if (ppPdpt)
1749	*ppPdpt = pPdpt;
1750
1751	/* Allocate page directory if not present. */
1752	EPTPDPTE Pdpe;
1753	Pdpe.u = pPdpe->u;
1754	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1755	{
1756	RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1757	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1758	pShwPage->idx, iPdPt, false /fLockPage/,
1759	&pShwPage);
1760	AssertRCReturn(rc, rc);
1761
1762	/* Hook up the new PD now. */
1763	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1764	}
1765	else
1766	{
1767	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1768	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1769
1770	pgmPoolCacheUsed(pPool, pShwPage);
1771
1772	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1773	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1774	{ }
1775	else
1776	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1777	}
1778
1779	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1780	return VINF_SUCCESS;
1781	}
1782
1783
1784	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1785	/**
1786	* Syncs the SHADOW nested-guest page directory pointer for the specified address.
1787	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1788	*
1789	* @returns VBox status code.
1790	* @param pVCpu The cross context virtual CPU structure.
1791	* @param GCPhysNested The nested-guest physical address.
1792	* @param ppPdpt Where to store the PDPT. Optional, can be NULL.
1793	* @param ppPD Where to store the PD. Optional, can be NULL.
1794	* @param pGstWalkAll The guest walk info.
1795	*/
1796	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
1797	PPGMPTWALKGST pGstWalkAll)
1798	{
1799	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1800	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1801	int rc;
1802
1803	PPGMPOOLPAGE pShwPage;
1804	Assert(pVM->pgm.s.fNestedPaging);
1805	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
1806	PGM_LOCK_ASSERT_OWNER(pVM);
1807
1808	/*
1809	* PML4 level.
1810	*/
1811	{
1812	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1813	Assert(pPml4);
1814
1815	/* Allocate page directory pointer table if not present. */
1816	{
1817	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
1818	const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1819	PEPTPML4E pPml4e = &pPml4->a[iPml4e];
1820
1821	if (!(pPml4e->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1822	{
1823	RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
1824	rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
1825	PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /fLockPage/,
1826	&pShwPage);
1827	AssertRCReturn(rc, rc);
1828
1829	/* Hook up the new PDPT now. */
1830	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1831	}
1832	else
1833	{
1834	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1835	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1836
1837	pgmPoolCacheUsed(pPool, pShwPage);
1838
1839	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1840	if (pPml4e->u != (pShwPage->Core.Key \| fShwFlags))
1841	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1842	}
1843	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1844	Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
1845	}
1846	}
1847
1848	/*
1849	* PDPT level.
1850	*/
1851	{
1852	AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
1853
1854	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1855	if (ppPdpt)
1856	*ppPdpt = pPdpt;
1857
1858	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
1859	const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1860	PEPTPDPTE pPdpte = &pPdpt->a[iPdPte];
1861
1862	if (!(pPdpte->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1863	{
1864	RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
1865	rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1866	pShwPage->idx, iPdPte, false /fLockPage/, &pShwPage);
1867	AssertRCReturn(rc, rc);
1868
1869	/* Hook up the new PD now. */
1870	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1871	}
1872	else
1873	{
1874	pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
1875	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1876
1877	pgmPoolCacheUsed(pPool, pShwPage);
1878
1879	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1880	if (pPdpte->u != (pShwPage->Core.Key \| fShwFlags))
1881	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1882	}
1883	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1884	Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
1885
1886	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1887	}
1888
1889	return VINF_SUCCESS;
1890	}
1891	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1892
1893
1894	# ifdef IN_RING0
1895	/**
1896	* Synchronizes a range of nested page table entries.
1897	*
1898	* The caller must own the PGM lock.
1899	*
1900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1901	* @param GCPhys Where to start.
1902	* @param cPages How many pages which entries should be synced.
1903	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1904	* host paging mode for AMD-V).
1905	*/
1906	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1907	{
1908	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1909
1910	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1911	int rc;
1912	switch (enmShwPagingMode)
1913	{
1914	case PGMMODE_32_BIT:
1915	{
1916	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1917	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1918	break;
1919	}
1920
1921	case PGMMODE_PAE:
1922	case PGMMODE_PAE_NX:
1923	{
1924	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1925	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1926	break;
1927	}
1928
1929	case PGMMODE_AMD64:
1930	case PGMMODE_AMD64_NX:
1931	{
1932	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1933	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1934	break;
1935	}
1936
1937	case PGMMODE_EPT:
1938	{
1939	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1940	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1941	break;
1942	}
1943
1944	default:
1945	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1946	}
1947	return rc;
1948	}
1949	# endif /* IN_RING0 */
1950
1951	# endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
1952
1953	#endif /* VBOX_VMM_TARGET_X86 */
1954
1955
1956	/**
1957	* Gets effective Guest OS page information.
1958	*
1959	* @returns VBox status code.
1960	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1961	* @param GCPtr Guest Context virtual address of the page.
1962	* @param pWalk Where to store the page walk information.
1963	* @thread EMT(pVCpu)
1964	*/
1965	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
1966	{
1967	VMCPU_ASSERT_EMT(pVCpu);
1968	Assert(pWalk);
1969	#ifdef VBOX_VMM_TARGET_X86
1970	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1971	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1972	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1973	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
1974
1975	#elif defined(VBOX_VMM_TARGET_ARMV8)
1976	return pgmGstGetPageArmv8Hack(pVCpu, GCPtr, pWalk);
1977	#else
1978	# error "port me"
1979	#endif
1980	}
1981
1982	#ifdef VBOX_VMM_TARGET_X86
1983
1984	/**
1985	* Gets effective Guest OS page information.
1986	*
1987	* @returns VBox status code.
1988	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1989	* @param GCPtr Guest Context virtual address of the page.
1990	* @param fFlags PGMQPAGE_F_XXX. If zero, no accessed or dirty bits will
1991	* be set.
1992	* @param pWalk Where to store the page walk information.
1993	* @thread EMT(pVCpu)
1994	*/
1995	VMM_INT_DECL(int) PGMGstQueryPageFast(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
1996	{
1997	VMCPU_ASSERT_EMT(pVCpu);
1998	Assert(pWalk);
1999	Assert(!(fFlags & ~(PGMQPAGE_F_VALID_MASK)));
2000	Assert(!(fFlags & PGMQPAGE_F_EXECUTE) \|\| !(fFlags & PGMQPAGE_F_WRITE));
2001	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2002	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2003	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
2004	return g_aPgmGuestModeData[idx].pfnQueryPageFast(pVCpu, GCPtr, fFlags, pWalk);
2005	}
2006
2007
2008	/**
2009	* Maps the guest CR3.
2010	*
2011	* @returns VBox status code.
2012	* @param pVCpu The cross context virtual CPU structure.
2013	* @param GCPhysCr3 The guest CR3 value.
2014	* @param pHCPtrGuestCr3 Where to store the mapped memory.
2015	*/
2016	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
2017	{
2018	/** @todo this needs some reworking wrt. locking? */
2019	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2020	PGM_LOCK_VOID(pVM);
2021	PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
2022	AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
2023
2024	RTHCPTR HCPtrGuestCr3;
2025	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
2026	PGM_UNLOCK(pVM);
2027
2028	*pHCPtrGuestCr3 = HCPtrGuestCr3;
2029	return rc;
2030	}
2031
2032
2033	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2034	/**
2035	* Unmaps the guest CR3.
2036	*
2037	* @returns VBox status code.
2038	* @param pVCpu The cross context virtual CPU structure.
2039	*/
2040	DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
2041	{
2042	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2043	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2044	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
2045	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
2046	}
2047	# endif
2048
2049	#endif /* VBOX_VMM_TARGET_X86 */
2050
2051
2052	/**
2053	* Performs a guest page table walk.
2054	*
2055	* The guest should be in paged protect mode or long mode when making a call to
2056	* this function.
2057	*
2058	* @returns VBox status code.
2059	* @retval VINF_SUCCESS on success.
2060	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2061	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2062	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2063	*
2064	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2065	* @param GCPtr The guest virtual address to walk by.
2066	* @param pWalk Where to return the walk result. This is valid for some
2067	* error codes as well.
2068	* @param pGstWalk The guest mode specific page walk information.
2069	*/
2070	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2071	{
2072	VMCPU_ASSERT_EMT(pVCpu);
2073	#ifdef VBOX_VMM_TARGET_X86
2074	switch (pVCpu->pgm.s.enmGuestMode)
2075	{
2076	case PGMMODE_32_BIT:
2077	pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
2078	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
2079
2080	case PGMMODE_PAE:
2081	case PGMMODE_PAE_NX:
2082	pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
2083	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
2084
2085	case PGMMODE_AMD64:
2086	case PGMMODE_AMD64_NX:
2087	pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
2088	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
2089
2090	case PGMMODE_REAL:
2091	case PGMMODE_PROTECTED:
2092	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2093	return VERR_PGM_NOT_USED_IN_MODE;
2094
2095	case PGMMODE_EPT:
2096	case PGMMODE_NESTED_32BIT:
2097	case PGMMODE_NESTED_PAE:
2098	case PGMMODE_NESTED_AMD64:
2099	default:
2100	AssertFailed();
2101	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2102	return VERR_PGM_NOT_USED_IN_MODE;
2103	}
2104
2105	#elif defined(VBOX_VMM_TARGET_ARMV8)
2106	/** @todo temporary hack. */
2107	RT_NOREF(pGstWalk);
2108	return pgmGstGetPageArmv8Hack(pVCpu, GCPtr, pWalk);
2109
2110	#else
2111	# error "port me"
2112	#endif
2113	}
2114
2115
2116	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2117	/**
2118	* Performs a guest second-level address translation (SLAT).
2119	*
2120	* @returns VBox status code.
2121	* @retval VINF_SUCCESS on success.
2122	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2123	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2124	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2125	*
2126	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2127	* @param GCPhysNested The nested-guest physical address being translated.
2128	* @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the
2129	* cause for this translation.
2130	* @param GCPtrNested The nested-guest virtual address that initiated the
2131	* SLAT. If none, pass 0 (and not NIL_RTGCPTR).
2132	* @param pWalk Where to return the walk result. This is updated for
2133	* all error codes other than
2134	* VERR_PGM_NOT_USED_IN_MODE.
2135	* @param pGstWalk Where to store the second-level paging-mode specific
2136	* walk info.
2137	*/
2138	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
2139	PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2140	{
2141	/* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
2142	Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
2143	&& pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
2144	AssertPtr(pWalk);
2145	AssertPtr(pGstWalk);
2146	switch (pVCpu->pgm.s.enmGuestSlatMode)
2147	{
2148	case PGMSLAT_EPT:
2149	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
2150	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
2151
2152	default:
2153	AssertFailed();
2154	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2155	return VERR_PGM_NOT_USED_IN_MODE;
2156	}
2157	}
2158	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
2159
2160
2161	/**
2162	* Tries to continue the previous walk.
2163	*
2164	* @note Requires the caller to hold the PGM lock from the first
2165	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
2166	* we cannot use the pointers.
2167	*
2168	* @returns VBox status code.
2169	* @retval VINF_SUCCESS on success.
2170	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2171	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2172	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2173	*
2174	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2175	* @param GCPtr The guest virtual address to walk by.
2176	* @param pWalk Pointer to the previous walk result and where to return
2177	* the result of this walk. This is valid for some error
2178	* codes as well.
2179	* @param pGstWalk The guest-mode specific walk information.
2180	*/
2181	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2182	{
2183	#ifdef VBOX_VMM_TARGET_X86 /** @todo optimize for ARMv8 */
2184	/*
2185	* We can only handle successfully walks.
2186	* We also limit ourselves to the next page.
2187	*/
2188	if ( pWalk->fSucceeded
2189	&& GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
2190	{
2191	Assert(pWalk->uLevel == 0);
2192	if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2193	{
2194	/*
2195	* AMD64
2196	*/
2197	if (!pWalk->fGigantPage && !pWalk->fBigPage)
2198	{
2199	/*
2200	* We fall back to full walk if the PDE table changes, if any
2201	* reserved bits are set, or if the effective page access changes.
2202	*/
2203	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
2204	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
2205	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
2206	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
2207
2208	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
2209	{
2210	if (pGstWalk->u.Amd64.pPte)
2211	{
2212	X86PTEPAE Pte;
2213	Pte.u = pGstWalk->u.Amd64.pPte[1].u;
2214	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2215	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2216	{
2217	pWalk->GCPtr = GCPtr;
2218	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2219	pGstWalk->u.Amd64.Pte.u = Pte.u;
2220	pGstWalk->u.Amd64.pPte++;
2221	return VINF_SUCCESS;
2222	}
2223	}
2224	}
2225	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
2226	{
2227	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2228	if (pGstWalk->u.Amd64.pPde)
2229	{
2230	X86PDEPAE Pde;
2231	Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2232	if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2233	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2234	{
2235	/* Get the new PTE and check out the first entry. */
2236	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2237	&pGstWalk->u.Amd64.pPt);
2238	if (RT_SUCCESS(rc))
2239	{
2240	pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2241	X86PTEPAE Pte;
2242	Pte.u = pGstWalk->u.Amd64.pPte->u;
2243	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2244	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2245	{
2246	pWalk->GCPtr = GCPtr;
2247	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2248	pGstWalk->u.Amd64.Pte.u = Pte.u;
2249	pGstWalk->u.Amd64.Pde.u = Pde.u;
2250	pGstWalk->u.Amd64.pPde++;
2251	return VINF_SUCCESS;
2252	}
2253	}
2254	}
2255	}
2256	}
2257	}
2258	else if (!pWalk->fGigantPage)
2259	{
2260	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2261	{
2262	pWalk->GCPtr = GCPtr;
2263	pWalk->GCPhys += GUEST_PAGE_SIZE;
2264	return VINF_SUCCESS;
2265	}
2266	}
2267	else
2268	{
2269	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2270	{
2271	pWalk->GCPtr = GCPtr;
2272	pWalk->GCPhys += GUEST_PAGE_SIZE;
2273	return VINF_SUCCESS;
2274	}
2275	}
2276	}
2277	}
2278	#endif /* VBOX_VMM_TARGET_X86 */
2279	/* Case we don't handle. Do full walk. */
2280	return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2281	}
2282
2283
2284	#ifdef VBOX_VMM_TARGET_X86
2285	/**
2286	* Modify page flags for a range of pages in the guest's tables
2287	*
2288	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2289	*
2290	* @returns VBox status code.
2291	* @param pVCpu The cross context virtual CPU structure.
2292	* @param GCPtr Virtual address of the first page in the range.
2293	* @param cb Size (in bytes) of the range to apply the modification to.
2294	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2295	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2296	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2297	*/
2298	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2299	{
2300	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2301	VMCPU_ASSERT_EMT(pVCpu);
2302
2303	/*
2304	* Validate input.
2305	*/
2306	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2307	Assert(cb);
2308
2309	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2310
2311	/*
2312	* Adjust input.
2313	*/
2314	cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2315	cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2316	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2317
2318	/*
2319	* Call worker.
2320	*/
2321	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2322	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2323	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2324	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2325
2326	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2327	return rc;
2328	}
2329	#endif /* VBOX_VMM_TARGET_X86 */
2330
2331	#ifdef VBOX_VMM_TARGET_X86
2332
2333	/**
2334	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2335	*
2336	* @returns @c true if the PDPE is valid, @c false otherwise.
2337	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2338	* @param paPaePdpes The PAE PDPEs to validate.
2339	*
2340	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2341	*/
2342	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2343	{
2344	Assert(paPaePdpes);
2345	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2346	{
2347	X86PDPE const PaePdpe = paPaePdpes[i];
2348	if ( !(PaePdpe.u & X86_PDPE_P)
2349	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2350	{ /* likely */ }
2351	else
2352	return false;
2353	}
2354	return true;
2355	}
2356
2357
2358	/**
2359	* Performs the lazy mapping of the 32-bit guest PD.
2360	*
2361	* @returns VBox status code.
2362	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2363	* @param ppPd Where to return the pointer to the mapping. This is
2364	* always set.
2365	*/
2366	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2367	{
2368	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2369	PGM_LOCK_VOID(pVM);
2370
2371	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2372
2373	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2374	PPGMPAGE pPage;
2375	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2376	if (RT_SUCCESS(rc))
2377	{
2378	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2379	if (RT_SUCCESS(rc))
2380	{
2381	# ifdef IN_RING3
2382	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2383	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2384	# else
2385	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2386	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2387	# endif
2388	PGM_UNLOCK(pVM);
2389	return VINF_SUCCESS;
2390	}
2391	AssertRC(rc);
2392	}
2393	PGM_UNLOCK(pVM);
2394
2395	*ppPd = NULL;
2396	return rc;
2397	}
2398
2399
2400	/**
2401	* Performs the lazy mapping of the PAE guest PDPT.
2402	*
2403	* @returns VBox status code.
2404	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2405	* @param ppPdpt Where to return the pointer to the mapping. This is
2406	* always set.
2407	*/
2408	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2409	{
2410	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2411	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2412	PGM_LOCK_VOID(pVM);
2413
2414	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2415	PPGMPAGE pPage;
2416	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2417	if (RT_SUCCESS(rc))
2418	{
2419	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2420	if (RT_SUCCESS(rc))
2421	{
2422	# ifdef IN_RING3
2423	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2424	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2425	# else
2426	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2427	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2428	# endif
2429	PGM_UNLOCK(pVM);
2430	return VINF_SUCCESS;
2431	}
2432	AssertRC(rc);
2433	}
2434
2435	PGM_UNLOCK(pVM);
2436	*ppPdpt = NULL;
2437	return rc;
2438	}
2439
2440
2441	/**
2442	* Performs the lazy mapping / updating of a PAE guest PD.
2443	*
2444	* @returns Pointer to the mapping.
2445	* @returns VBox status code.
2446	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2447	* @param iPdpt Which PD entry to map (0..3).
2448	* @param ppPd Where to return the pointer to the mapping. This is
2449	* always set.
2450	*/
2451	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2452	{
2453	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2454	PGM_LOCK_VOID(pVM);
2455
2456	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2457	Assert(pGuestPDPT);
2458	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2459	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2460	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2461
2462	PPGMPAGE pPage;
2463	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2464	if (RT_SUCCESS(rc))
2465	{
2466	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2467	AssertRC(rc);
2468	if (RT_SUCCESS(rc))
2469	{
2470	# ifdef IN_RING3
2471	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2472	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2473	# else
2474	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2475	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2476	# endif
2477	if (fChanged)
2478	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2479	PGM_UNLOCK(pVM);
2480	return VINF_SUCCESS;
2481	}
2482	}
2483
2484	/* Invalid page or some failure, invalidate the entry. */
2485	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2486	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2487	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2488
2489	PGM_UNLOCK(pVM);
2490	return rc;
2491	}
2492
2493
2494	/**
2495	* Performs the lazy mapping of the 32-bit guest PD.
2496	*
2497	* @returns VBox status code.
2498	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2499	* @param ppPml4 Where to return the pointer to the mapping. This will
2500	* always be set.
2501	*/
2502	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2503	{
2504	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2505	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2506	PGM_LOCK_VOID(pVM);
2507
2508	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2509	PPGMPAGE pPage;
2510	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2511	if (RT_SUCCESS(rc))
2512	{
2513	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2514	if (RT_SUCCESS(rc))
2515	{
2516	# ifdef IN_RING3
2517	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2518	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2519	# else
2520	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2521	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2522	# endif
2523	PGM_UNLOCK(pVM);
2524	return VINF_SUCCESS;
2525	}
2526	}
2527
2528	PGM_UNLOCK(pVM);
2529	*ppPml4 = NULL;
2530	return rc;
2531	}
2532
2533
2534	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2535	/**
2536	* Performs the lazy mapping of the guest PML4 table when using EPT paging.
2537	*
2538	* @returns VBox status code.
2539	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2540	* @param ppEptPml4 Where to return the pointer to the mapping. This will
2541	* always be set.
2542	*/
2543	int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2544	{
2545	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2546	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2547	PGM_LOCK_VOID(pVM);
2548
2549	RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2550	PPGMPAGE pPage;
2551	int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2552	if (RT_SUCCESS(rc))
2553	{
2554	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2555	if (RT_SUCCESS(rc))
2556	{
2557	# ifdef IN_RING3
2558	pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2559	pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2560	# else
2561	pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2562	pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2563	# endif
2564	PGM_UNLOCK(pVM);
2565	return VINF_SUCCESS;
2566	}
2567	}
2568
2569	PGM_UNLOCK(pVM);
2570	*ppEptPml4 = NULL;
2571	return rc;
2572	}
2573	# endif
2574
2575
2576	/**
2577	* Gets the current CR3 register value for the shadow memory context.
2578	* @returns CR3 value.
2579	* @param pVCpu The cross context virtual CPU structure.
2580	*/
2581	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2582	{
2583	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2584	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2585	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2586	return pPoolPage->Core.Key;
2587	# else
2588	RT_NOREF(pVCpu);
2589	return NIL_RTHCPHYS;
2590	# endif
2591	}
2592
2593
2594	/**
2595	* Forces lazy remapping of the guest's PAE page-directory structures.
2596	*
2597	* @param pVCpu The cross context virtual CPU structure.
2598	*/
2599	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2600	{
2601	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2602	{
2603	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2604	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2605	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2606	}
2607	}
2608
2609
2610	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2611	/**
2612	* Performs second-level address translation for the given CR3 and updates the
2613	* nested-guest CR3 when successful.
2614	*
2615	* @returns VBox status code.
2616	* @param pVCpu The cross context virtual CPU structure.
2617	* @param uCr3 The masked nested-guest CR3 value.
2618	* @param pGCPhysCR3 Where to store the translated CR3.
2619	*
2620	* @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2621	* mindful of this in code that's hyper sensitive to the order of
2622	* operations.
2623	*/
2624	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2625	{
2626	if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2627	{
2628	PGMPTWALK Walk;
2629	PGMPTWALKGST GstWalk;
2630	int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk, &GstWalk);
2631	if (RT_SUCCESS(rc))
2632	{
2633	/* Update nested-guest CR3. */
2634	pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2635
2636	/* Pass back the translated result. */
2637	*pGCPhysCr3 = Walk.GCPhys;
2638	return VINF_SUCCESS;
2639	}
2640
2641	/* Translation failed. */
2642	*pGCPhysCr3 = NIL_RTGCPHYS;
2643	return rc;
2644	}
2645
2646	/*
2647	* If the nested-guest CR3 has not changed, then the previously
2648	* translated CR3 result (i.e. GCPhysCR3) is passed back.
2649	*/
2650	*pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2651	return VINF_SUCCESS;
2652	}
2653	# endif
2654
2655
2656	/**
2657	* Performs and schedules necessary updates following a CR3 load or reload.
2658	*
2659	* This will normally involve mapping the guest PD or nPDPT
2660	*
2661	* @returns VBox status code.
2662	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2663	* safely be ignored and overridden since the FF will be set too then.
2664	* @param pVCpu The cross context virtual CPU structure.
2665	* @param cr3 The new cr3.
2666	* @param fGlobal Indicates whether this is a global flush or not.
2667	*/
2668	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2669	{
2670	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2671
2672	VMCPU_ASSERT_EMT(pVCpu);
2673
2674	/*
2675	* Always flag the necessary updates; necessary for hardware acceleration
2676	*/
2677	/** @todo optimize this, it shouldn't always be necessary. */
2678	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2679	if (fGlobal)
2680	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2681
2682	/*
2683	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2684	*/
2685	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2686	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2687	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2688	if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2689	&& PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2690	{
2691	RTGCPHYS GCPhysOut;
2692	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2693	if (RT_SUCCESS(rc))
2694	GCPhysCR3 = GCPhysOut;
2695	else
2696	{
2697	/* CR3 SLAT translation failed but we try to pretend it
2698	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2699	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2700	int const rc2 = pgmGstUnmapCr3(pVCpu);
2701	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2702	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2703	return rc2;
2704	}
2705	}
2706	#endif
2707
2708	LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2709	int rc = VINF_SUCCESS;
2710	if (GCPhysOldCR3 != GCPhysCR3)
2711	{
2712	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2713	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2714	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2715
2716	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2717	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2718	if (RT_LIKELY(rc == VINF_SUCCESS))
2719	{ }
2720	else
2721	{
2722	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2723	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2724	pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2725	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2726	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2727	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2728	}
2729
2730	if (fGlobal)
2731	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2732	else
2733	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2734	}
2735	else
2736	{
2737	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2738	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
2739	PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
2740	if (pPool->cDirtyPages)
2741	{
2742	PGM_LOCK_VOID(pVM);
2743	pgmPoolResetDirtyPages(pVM);
2744	PGM_UNLOCK(pVM);
2745	}
2746	#endif
2747	if (fGlobal)
2748	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2749	else
2750	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2751
2752	/*
2753	* Flush PAE PDPTEs.
2754	*/
2755	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2756	pgmGstFlushPaePdpes(pVCpu);
2757	}
2758
2759	if (!fGlobal)
2760	IEMTlbInvalidateAll(pVCpu);
2761	else
2762	IEMTlbInvalidateAllGlobal(pVCpu);
2763	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2764	return rc;
2765	}
2766
2767
2768	/**
2769	* Performs and schedules necessary updates following a CR3 load or reload when
2770	* using nested or extended paging.
2771	*
2772	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2773	* TLB and triggering a SyncCR3.
2774	*
2775	* This will normally involve mapping the guest PD or nPDPT
2776	*
2777	* @returns VBox status code.
2778	* @retval VINF_SUCCESS.
2779	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2780	* paging modes). This can safely be ignored and overridden since the
2781	* FF will be set too then.
2782	* @param pVCpu The cross context virtual CPU structure.
2783	* @param cr3 The new CR3.
2784	*/
2785	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2786	{
2787	VMCPU_ASSERT_EMT(pVCpu);
2788
2789	/* We assume we're only called in nested paging mode. */
2790	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2791
2792	/*
2793	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2794	*/
2795	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2796	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2797	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2798	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2799	{
2800	RTGCPHYS GCPhysOut;
2801	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2802	if (RT_SUCCESS(rc))
2803	GCPhysCR3 = GCPhysOut;
2804	else
2805	{
2806	/* CR3 SLAT translation failed but we try to pretend it
2807	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2808	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2809	int const rc2 = pgmGstUnmapCr3(pVCpu);
2810	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2811	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2812	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2813	return rc2;
2814	}
2815	}
2816	#endif
2817
2818	LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2819	int rc = VINF_SUCCESS;
2820	if (GCPhysOldCR3 != GCPhysCR3)
2821	{
2822	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2823	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2824	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2825
2826	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2827	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2828
2829	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2830	}
2831	/*
2832	* Flush PAE PDPTEs.
2833	*/
2834	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2835	pgmGstFlushPaePdpes(pVCpu);
2836
2837	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2838	return rc;
2839	}
2840
2841
2842	/**
2843	* Synchronize the paging structures.
2844	*
2845	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2846	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2847	* in several places, most importantly whenever the CR3 is loaded.
2848	*
2849	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2850	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2851	* the VMM into guest context.
2852	* @param pVCpu The cross context virtual CPU structure.
2853	* @param cr0 Guest context CR0 register
2854	* @param cr3 Guest context CR3 register
2855	* @param cr4 Guest context CR4 register
2856	* @param fGlobal Including global page directories or not
2857	*/
2858	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2859	{
2860	VMCPU_ASSERT_EMT(pVCpu);
2861
2862	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2863	/*
2864	* The pool may have pending stuff and even require a return to ring-3 to
2865	* clear the whole thing.
2866	*/
2867	int rcPool = pgmPoolSyncCR3(pVCpu);
2868	if (rcPool != VINF_SUCCESS)
2869	return rcPool;
2870	#endif
2871
2872	/*
2873	* We might be called when we shouldn't.
2874	*
2875	* The mode switching will ensure that the PD is resynced after every mode
2876	* switch. So, if we find ourselves here when in protected or real mode
2877	* we can safely clear the FF and return immediately.
2878	*/
2879	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2880	{
2881	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2882	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2883	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2884	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2885	return VINF_SUCCESS;
2886	}
2887
2888	/* If global pages are not supported, then all flushes are global. */
2889	if (!(cr4 & X86_CR4_PGE))
2890	fGlobal = true;
2891	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2892	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2893
2894	/*
2895	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2896	* This should be done before SyncCR3.
2897	*/
2898	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2899	{
2900	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2901
2902	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2903	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2904	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2905	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2906	{
2907	RTGCPHYS GCPhysOut;
2908	int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2909	if (RT_SUCCESS(rc2))
2910	GCPhysCR3 = GCPhysOut;
2911	else
2912	{
2913	/* CR3 SLAT translation failed but we try to pretend it
2914	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2915	AssertMsgFailed(("Failed to translate CR3 %#RX64. rc2=%Rrc\n", cr3, rc2));
2916	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2917	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2918	return rc2;
2919	}
2920	}
2921	#endif
2922	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2923	int rc = VINF_SUCCESS;
2924	if (GCPhysOldCR3 != GCPhysCR3)
2925	{
2926	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2927	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2928	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2929	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2930	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2931	}
2932
2933	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2934	if ( rc == VINF_PGM_SYNC_CR3
2935	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2936	{
2937	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2938	#ifdef IN_RING3
2939	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2940	rc = pgmPoolSyncCR3(pVCpu);
2941	# else
2942	rc = VINF_SUCCESS;
2943	# endif
2944	#else
2945	if (rc == VINF_PGM_SYNC_CR3)
2946	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2947	return VINF_PGM_SYNC_CR3;
2948	#endif
2949	}
2950	AssertRCReturn(rc, rc);
2951	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2952	}
2953
2954	/*
2955	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2956	*/
2957	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2958
2959	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2960	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2961	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2962	int rcSync = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2963
2964	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2965	AssertMsg(rcSync == VINF_SUCCESS \|\| rcSync == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rcSync), ("rcSync=%Rrc\n", rcSync));
2966	if (rcSync == VINF_SUCCESS)
2967	{
2968	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2969	{
2970	/* Go back to ring 3 if a pgm pool sync is again pending. */
2971	return VINF_PGM_SYNC_CR3;
2972	}
2973
2974	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2975	{
2976	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2977	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2978	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2979	}
2980	}
2981
2982	/*
2983	* Now flush the CR3 (guest context).
2984	*/
2985	if (rcSync == VINF_SUCCESS)
2986	PGM_INVL_VCPU_TLBS(pVCpu);
2987	return rcSync;
2988	}
2989
2990
2991	/**
2992	* Maps all the PAE PDPE entries.
2993	*
2994	* @returns VBox status code.
2995	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2996	* @param paPaePdpes The new PAE PDPE values.
2997	*
2998	* @remarks This function may be invoked during the process of changing the guest
2999	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
3000	* reflect PAE paging just yet.
3001	*/
3002	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
3003	{
3004	Assert(paPaePdpes);
3005	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3006	{
3007	X86PDPE const PaePdpe = paPaePdpes[i];
3008
3009	/*
3010	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
3011	* are deferred.[1] Also, different situations require different handling of invalid
3012	* PDPE entries. Here we assume the caller has already validated or doesn't require
3013	* validation of the PDPEs.
3014	*
3015	* In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
3016	* validated by the VMX transition.
3017	*
3018	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
3019	*/
3020	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
3021	{
3022	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3023	RTHCPTR HCPtr;
3024
3025	RTGCPHYS GCPhys;
3026	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3027	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3028	{
3029	PGMPTWALK Walk;
3030	PGMPTWALKGST GstWalk;
3031	RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
3032	int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid /, 0 / GCPtrNested */,
3033	&Walk, &GstWalk);
3034	if (RT_SUCCESS(rc))
3035	GCPhys = Walk.GCPhys;
3036	else
3037	{
3038	/*
3039	* Second-level address translation of the PAE PDPE has failed but we must -NOT-
3040	* abort and return a failure now. This is because we're called from a Mov CRx
3041	* instruction (or similar operation). Let's just pretend success but flag that
3042	* we need to map this PDPE lazily later.
3043	*
3044	* See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
3045	* See Intel spec. 28.3.1 "EPT Overview".
3046	*/
3047	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3048	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3049	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3050	continue;
3051	}
3052	}
3053	else
3054	#endif
3055	{
3056	GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
3057	}
3058
3059	PGM_LOCK_VOID(pVM);
3060	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
3061	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
3062	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
3063	PGM_UNLOCK(pVM);
3064	if (RT_SUCCESS(rc))
3065	{
3066	#ifdef IN_RING3
3067	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
3068	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
3069	#else
3070	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
3071	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
3072	#endif
3073	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
3074	continue;
3075	}
3076	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
3077	}
3078	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3079	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3080	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3081	}
3082	return VINF_SUCCESS;
3083	}
3084
3085
3086	/**
3087	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
3088	*
3089	* @returns VBox status code.
3090	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3091	* @param cr3 The guest CR3 value.
3092	*
3093	* @remarks This function may be invoked during the process of changing the guest
3094	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
3095	* PAE paging just yet.
3096	*/
3097	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
3098	{
3099	/*
3100	* Read the page-directory-pointer table (PDPT) at CR3.
3101	*/
3102	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
3103	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
3104
3105	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3106	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3107	{
3108	RTGCPHYS GCPhysOut;
3109	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
3110	if (RT_SUCCESS(rc))
3111	GCPhysCR3 = GCPhysOut;
3112	else
3113	{
3114	Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
3115	return rc;
3116	}
3117	}
3118	#endif
3119
3120	RTHCPTR HCPtrGuestCr3;
3121	int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
3122	if (RT_SUCCESS(rc))
3123	{
3124	/*
3125	* Validate the page-directory-pointer table entries (PDPE).
3126	*/
3127	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
3128	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
3129	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
3130	{
3131	/*
3132	* Map the PDPT.
3133	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
3134	* that PGMFlushTLB will be called soon and only a change to CR3 then
3135	* will cause the shadow page tables to be updated.
3136	*/
3137	#ifdef IN_RING3
3138	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
3139	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
3140	#else
3141	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
3142	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
3143	#endif
3144
3145	/*
3146	* Update CPUM and map the 4 PAE PDPEs.
3147	*/
3148	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
3149	rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
3150	if (RT_SUCCESS(rc))
3151	{
3152	#ifdef IN_RING3
3153	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
3154	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
3155	#else
3156	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
3157	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
3158	#endif
3159	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
3160	}
3161	}
3162	else
3163	rc = VERR_PGM_PAE_PDPE_RSVD;
3164	}
3165	return rc;
3166	}
3167
3168
3169	/**
3170	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
3171	*
3172	* @returns VBox status code, with the following informational code for
3173	* VM scheduling.
3174	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
3175	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
3176	*
3177	* @param pVCpu The cross context virtual CPU structure.
3178	* @param cr0 The new cr0.
3179	* @param cr4 The new cr4.
3180	* @param efer The new extended feature enable register.
3181	* @param fForce Whether to force a mode change.
3182	*/
3183	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
3184	{
3185	VMCPU_ASSERT_EMT(pVCpu);
3186
3187	/*
3188	* Calc the new guest mode.
3189	*
3190	* Note! We check PG before PE and without requiring PE because of the
3191	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
3192	*/
3193	PGMMODE enmGuestMode;
3194	if (cr0 & X86_CR0_PG)
3195	{
3196	if (!(cr4 & X86_CR4_PAE))
3197	{
3198	bool const fPse = !!(cr4 & X86_CR4_PSE);
3199	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
3200	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
3201	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
3202	enmGuestMode = PGMMODE_32_BIT;
3203	}
3204	else if (!(efer & MSR_K6_EFER_LME))
3205	{
3206	if (!(efer & MSR_K6_EFER_NXE))
3207	enmGuestMode = PGMMODE_PAE;
3208	else
3209	enmGuestMode = PGMMODE_PAE_NX;
3210	}
3211	else
3212	{
3213	if (!(efer & MSR_K6_EFER_NXE))
3214	enmGuestMode = PGMMODE_AMD64;
3215	else
3216	enmGuestMode = PGMMODE_AMD64_NX;
3217	}
3218	}
3219	else if (!(cr0 & X86_CR0_PE))
3220	enmGuestMode = PGMMODE_REAL;
3221	else
3222	enmGuestMode = PGMMODE_PROTECTED;
3223
3224	/*
3225	* Did it change?
3226	*/
3227	if ( !fForce
3228	&& pVCpu->pgm.s.enmGuestMode == enmGuestMode)
3229	return VINF_SUCCESS;
3230
3231	/* Flush the TLB */
3232	PGM_INVL_VCPU_TLBS(pVCpu);
3233	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3234	}
3235
3236
3237	/**
3238	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3239	*
3240	* @returns PGM_TYPE_*.
3241	* @param pgmMode The mode value to convert.
3242	*/
3243	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3244	{
3245	switch (pgmMode)
3246	{
3247	case PGMMODE_REAL: return PGM_TYPE_REAL;
3248	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3249	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3250	case PGMMODE_PAE:
3251	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3252	case PGMMODE_AMD64:
3253	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3254	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3255	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3256	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3257	case PGMMODE_EPT: return PGM_TYPE_EPT;
3258	case PGMMODE_NONE: return PGM_TYPE_NONE;
3259	default:
3260	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3261	}
3262	}
3263
3264
3265	/**
3266	* Calculates the shadow paging mode.
3267	*
3268	* @returns The shadow paging mode.
3269	* @param pVM The cross context VM structure.
3270	* @param enmGuestMode The guest mode.
3271	* @param enmHostMode The host mode.
3272	* @param enmShadowMode The current shadow mode.
3273	*/
3274	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3275	{
3276	switch (enmGuestMode)
3277	{
3278	case PGMMODE_REAL:
3279	case PGMMODE_PROTECTED:
3280	switch (enmHostMode)
3281	{
3282	case SUPPAGINGMODE_32_BIT:
3283	case SUPPAGINGMODE_32_BIT_GLOBAL:
3284	enmShadowMode = PGMMODE_32_BIT;
3285	break;
3286
3287	case SUPPAGINGMODE_PAE:
3288	case SUPPAGINGMODE_PAE_NX:
3289	case SUPPAGINGMODE_PAE_GLOBAL:
3290	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3291	enmShadowMode = PGMMODE_PAE;
3292	break;
3293
3294	case SUPPAGINGMODE_AMD64:
3295	case SUPPAGINGMODE_AMD64_GLOBAL:
3296	case SUPPAGINGMODE_AMD64_NX:
3297	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3298	enmShadowMode = PGMMODE_PAE;
3299	break;
3300
3301	default:
3302	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3303	}
3304	break;
3305
3306	case PGMMODE_32_BIT:
3307	switch (enmHostMode)
3308	{
3309	case SUPPAGINGMODE_32_BIT:
3310	case SUPPAGINGMODE_32_BIT_GLOBAL:
3311	enmShadowMode = PGMMODE_32_BIT;
3312	break;
3313
3314	case SUPPAGINGMODE_PAE:
3315	case SUPPAGINGMODE_PAE_NX:
3316	case SUPPAGINGMODE_PAE_GLOBAL:
3317	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3318	enmShadowMode = PGMMODE_PAE;
3319	break;
3320
3321	case SUPPAGINGMODE_AMD64:
3322	case SUPPAGINGMODE_AMD64_GLOBAL:
3323	case SUPPAGINGMODE_AMD64_NX:
3324	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3325	enmShadowMode = PGMMODE_PAE;
3326	break;
3327
3328	default:
3329	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3330	}
3331	break;
3332
3333	case PGMMODE_PAE:
3334	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3335	switch (enmHostMode)
3336	{
3337	case SUPPAGINGMODE_32_BIT:
3338	case SUPPAGINGMODE_32_BIT_GLOBAL:
3339	enmShadowMode = PGMMODE_PAE;
3340	break;
3341
3342	case SUPPAGINGMODE_PAE:
3343	case SUPPAGINGMODE_PAE_NX:
3344	case SUPPAGINGMODE_PAE_GLOBAL:
3345	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3346	enmShadowMode = PGMMODE_PAE;
3347	break;
3348
3349	case SUPPAGINGMODE_AMD64:
3350	case SUPPAGINGMODE_AMD64_GLOBAL:
3351	case SUPPAGINGMODE_AMD64_NX:
3352	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3353	enmShadowMode = PGMMODE_PAE;
3354	break;
3355
3356	default:
3357	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3358	}
3359	break;
3360
3361	case PGMMODE_AMD64:
3362	case PGMMODE_AMD64_NX:
3363	switch (enmHostMode)
3364	{
3365	case SUPPAGINGMODE_32_BIT:
3366	case SUPPAGINGMODE_32_BIT_GLOBAL:
3367	enmShadowMode = PGMMODE_AMD64;
3368	break;
3369
3370	case SUPPAGINGMODE_PAE:
3371	case SUPPAGINGMODE_PAE_NX:
3372	case SUPPAGINGMODE_PAE_GLOBAL:
3373	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3374	enmShadowMode = PGMMODE_AMD64;
3375	break;
3376
3377	case SUPPAGINGMODE_AMD64:
3378	case SUPPAGINGMODE_AMD64_GLOBAL:
3379	case SUPPAGINGMODE_AMD64_NX:
3380	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3381	enmShadowMode = PGMMODE_AMD64;
3382	break;
3383
3384	default:
3385	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3386	}
3387	break;
3388
3389	default:
3390	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3391	}
3392
3393	/*
3394	* Override the shadow mode when NEM, IEM or nested paging is active.
3395	*/
3396	if (!VM_IS_HM_ENABLED(pVM))
3397	{
3398	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
3399	pVM->pgm.s.fNestedPaging = true;
3400	enmShadowMode = PGMMODE_NONE;
3401	}
3402	else
3403	{
3404	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3405	pVM->pgm.s.fNestedPaging = fNestedPaging;
3406	if (fNestedPaging)
3407	{
3408	if (HMIsVmxActive(pVM))
3409	enmShadowMode = PGMMODE_EPT;
3410	else
3411	{
3412	/* The nested SVM paging depends on the host one. */
3413	Assert(HMIsSvmActive(pVM));
3414	if ( enmGuestMode == PGMMODE_AMD64
3415	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3416	enmShadowMode = PGMMODE_NESTED_AMD64;
3417	else
3418	switch (pVM->pgm.s.enmHostMode)
3419	{
3420	case SUPPAGINGMODE_32_BIT:
3421	case SUPPAGINGMODE_32_BIT_GLOBAL:
3422	enmShadowMode = PGMMODE_NESTED_32BIT;
3423	break;
3424
3425	case SUPPAGINGMODE_PAE:
3426	case SUPPAGINGMODE_PAE_GLOBAL:
3427	case SUPPAGINGMODE_PAE_NX:
3428	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3429	enmShadowMode = PGMMODE_NESTED_PAE;
3430	break;
3431
3432	case SUPPAGINGMODE_AMD64:
3433	case SUPPAGINGMODE_AMD64_GLOBAL:
3434	case SUPPAGINGMODE_AMD64_NX:
3435	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3436	enmShadowMode = PGMMODE_NESTED_AMD64;
3437	break;
3438
3439	default:
3440	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3441	}
3442	}
3443	}
3444	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3445	else
3446	{
3447	/* Nested paging is a requirement for nested VT-x. */
3448	AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3449	}
3450	#endif
3451	}
3452
3453	return enmShadowMode;
3454	}
3455
3456	#endif /* VBOX_VMM_TARGET_X86 */
3457
3458	/**
3459	* Performs the actual mode change.
3460	* This is called by PGMChangeMode and pgmR3InitPaging().
3461	*
3462	* @returns VBox status code. May suspend or power off the VM on error, but this
3463	* will trigger using FFs and not informational status codes.
3464	*
3465	* @param pVM The cross context VM structure.
3466	* @param pVCpu The cross context virtual CPU structure.
3467	* @param enmGuestMode The new guest mode. This is assumed to be different from
3468	* the current mode.
3469	* @param fForce Whether to force a shadow paging mode change.
3470	*/
3471	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3472	{
3473	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3474	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3475
3476	#ifdef VBOX_VMM_TARGET_X86
3477	/*
3478	* Calc the shadow mode and switcher.
3479	*/
3480	PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3481	bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode \|\| fForce;
3482
3483	/*
3484	* Exit old mode(s).
3485	*/
3486	/* shadow */
3487	if (fShadowModeChanged)
3488	{
3489	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3490	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3491	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3492	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3493	{
3494	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3495	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3496	}
3497	}
3498	else
3499	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3500
3501	/* guest */
3502	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3503	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3504	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3505	{
3506	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3507	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3508	}
3509	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3510	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3511	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3512	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3513
3514	/*
3515	* Change the paging mode data indexes.
3516	*/
3517	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3518	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3519	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3520	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3521	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3522	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3523	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3524	# ifdef IN_RING3
3525	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3526	# endif
3527
3528	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3529	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3530	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3531	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3532	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3533	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3534	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3535	# ifdef IN_RING3
3536	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3537	# endif
3538
3539	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3540	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3541	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3542	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3543	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3544	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3545	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3546	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3547	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3548	# ifdef VBOX_STRICT
3549	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3550	# endif
3551
3552	/*
3553	* Determine SLAT mode -before- entering the new shadow mode!
3554	*/
3555	pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
3556
3557	/*
3558	* Enter new shadow mode (if changed).
3559	*/
3560	if (fShadowModeChanged)
3561	{
3562	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3563	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
3564	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3565	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3566	}
3567
3568	/*
3569	* Always flag the necessary updates
3570	*/
3571	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3572
3573	/*
3574	* Enter the new guest and shadow+guest modes.
3575	*/
3576	/* Calc the new CR3 value. */
3577	RTGCPHYS GCPhysCR3;
3578	switch (enmGuestMode)
3579	{
3580	case PGMMODE_REAL:
3581	case PGMMODE_PROTECTED:
3582	GCPhysCR3 = NIL_RTGCPHYS;
3583	break;
3584
3585	case PGMMODE_32_BIT:
3586	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3587	break;
3588
3589	case PGMMODE_PAE_NX:
3590	case PGMMODE_PAE:
3591	if (!pVM->cpum.ro.GuestFeatures.fPae)
3592	# ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3593	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3594	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3595	# else
3596	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3597
3598	# endif
3599	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3600	break;
3601
3602	# ifdef VBOX_WITH_64_BITS_GUESTS
3603	case PGMMODE_AMD64_NX:
3604	case PGMMODE_AMD64:
3605	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3606	break;
3607	# endif
3608	default:
3609	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3610	}
3611
3612	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3613	/*
3614	* If a nested-guest is using EPT paging:
3615	* - Update the second-level address translation (SLAT) mode.
3616	* - Indicate that the CR3 is nested-guest physical address.
3617	*/
3618	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3619	{
3620	if (PGMMODE_WITH_PAGING(enmGuestMode))
3621	{
3622	/*
3623	* Translate CR3 to its guest-physical address.
3624	* We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3625	* switching modes to keep it consistent with how GCPhysCR3 is updated.
3626	*/
3627	PGMPTWALK Walk;
3628	PGMPTWALKGST GstWalk;
3629	int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk,
3630	&GstWalk);
3631	if (RT_SUCCESS(rc))
3632	{ /* likely */ }
3633	else
3634	{
3635	/*
3636	* SLAT failed but we avoid reporting this to the caller because the caller
3637	* is not supposed to fail. The only time the caller needs to indicate a
3638	* failure to software is when PAE paging is used by the nested-guest, but
3639	* we handle the PAE case separately (e.g., see VMX transition in IEM).
3640	* In all other cases, the failure will be indicated when CR3 tries to be
3641	* translated on the next linear-address memory access.
3642	* See Intel spec. 27.2.1 "EPT Overview".
3643	*/
3644	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3645
3646	/* Trying to coax PGM to succeed for the time being... */
3647	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3648	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3649	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3650	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3651	return VINF_SUCCESS;
3652	}
3653	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3654	GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
3655	}
3656	}
3657	else
3658	Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3659	# endif
3660
3661	/*
3662	* Enter the new guest mode.
3663	*/
3664	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3665	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3666	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3667
3668	/* Set the new guest CR3 (and nested-guest CR3). */
3669	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3670
3671	/* status codes. */
3672	AssertRC(rc);
3673	AssertRC(rc2);
3674	if (RT_SUCCESS(rc))
3675	{
3676	rc = rc2;
3677	if (RT_SUCCESS(rc)) /* no informational status codes. */
3678	rc = VINF_SUCCESS;
3679	}
3680
3681	/*
3682	* Notify HM.
3683	*/
3684	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3685	return rc;
3686
3687	#elif defined(VBOX_VMM_TARGET_ARMV8)
3688	//AssertReleaseFailed(); /** @todo Called by the PGM saved state code. */
3689	RT_NOREF(pVM, pVCpu, enmGuestMode, fForce);
3690	return VINF_SUCCESS;
3691
3692	#else
3693	# error "port me"
3694	#endif
3695	}
3696
3697
3698	#ifdef VBOX_VMM_TARGET_X86
3699	/**
3700	* Called by CPUM or REM when CR0.WP changes to 1.
3701	*
3702	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3703	* @thread EMT
3704	*/
3705	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3706	{
3707	/*
3708	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3709	*
3710	* Use the counter to judge whether there might be pool pages with active
3711	* hacks in them. If there are, we will be running the risk of messing up
3712	* the guest by allowing it to write to read-only pages. Thus, we have to
3713	* clear the page pool ASAP if there is the slightest chance.
3714	*/
3715	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3716	{
3717	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3718
3719	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3720	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3721	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3722	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3723	}
3724	}
3725	#endif /* VBOX_VMM_TARGET_X86 */
3726
3727
3728	/**
3729	* Gets the current guest paging mode.
3730	*
3731	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3732	*
3733	* @returns The current paging mode.
3734	* @param pVCpu The cross context virtual CPU structure.
3735	*/
3736	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3737	{
3738	return pVCpu->pgm.s.enmGuestMode;
3739	}
3740
3741
3742	/**
3743	* Gets the current shadow paging mode.
3744	*
3745	* @returns The current paging mode.
3746	* @param pVCpu The cross context virtual CPU structure.
3747	*/
3748	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3749	{
3750	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3751	return pVCpu->pgm.s.enmShadowMode;
3752	#else
3753	RT_NOREF(pVCpu);
3754	return PGMMODE_NONE;
3755	#endif
3756	}
3757
3758
3759	#ifdef VBOX_VMM_TARGET_X86
3760	/**
3761	* Gets the current host paging mode.
3762	*
3763	* @returns The current paging mode.
3764	* @param pVM The cross context VM structure.
3765	*/
3766	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3767	{
3768	switch (pVM->pgm.s.enmHostMode)
3769	{
3770	case SUPPAGINGMODE_32_BIT:
3771	case SUPPAGINGMODE_32_BIT_GLOBAL:
3772	return PGMMODE_32_BIT;
3773
3774	case SUPPAGINGMODE_PAE:
3775	case SUPPAGINGMODE_PAE_GLOBAL:
3776	return PGMMODE_PAE;
3777
3778	case SUPPAGINGMODE_PAE_NX:
3779	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3780	return PGMMODE_PAE_NX;
3781
3782	case SUPPAGINGMODE_AMD64:
3783	case SUPPAGINGMODE_AMD64_GLOBAL:
3784	return PGMMODE_AMD64;
3785
3786	case SUPPAGINGMODE_AMD64_NX:
3787	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3788	return PGMMODE_AMD64_NX;
3789
3790	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3791	}
3792
3793	return PGMMODE_INVALID;
3794	}
3795	#endif /* VBOX_VMM_TARGET_X86 */
3796
3797
3798	/**
3799	* Get mode name.
3800	*
3801	* @returns read-only name string.
3802	* @param enmMode The mode which name is desired.
3803	*/
3804	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3805	{
3806	switch (enmMode)
3807	{
3808	case PGMMODE_REAL: return "Real";
3809	case PGMMODE_PROTECTED: return "Protected";
3810	case PGMMODE_32_BIT: return "32-bit";
3811	case PGMMODE_PAE: return "PAE";
3812	case PGMMODE_PAE_NX: return "PAE+NX";
3813	case PGMMODE_AMD64: return "AMD64";
3814	case PGMMODE_AMD64_NX: return "AMD64+NX";
3815	case PGMMODE_NESTED_32BIT: return "Nested-32";
3816	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3817	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3818	case PGMMODE_EPT: return "EPT";
3819	case PGMMODE_NONE: return "None";
3820	case PGMMODE_VMSA_V8_32: return "VMSAv8-32";
3821	case PGMMODE_VMSA_V8_64: return "VMSAv8-64";
3822	default: return "unknown mode value";
3823	}
3824	}
3825
3826
3827	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3828	/**
3829	* Gets the SLAT mode name.
3830	*
3831	* @returns The read-only SLAT mode descriptive string.
3832	* @param enmSlatMode The SLAT mode value.
3833	*/
3834	VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3835	{
3836	switch (enmSlatMode)
3837	{
3838	case PGMSLAT_DIRECT: return "Direct";
3839	case PGMSLAT_EPT: return "EPT";
3840	case PGMSLAT_32BIT: return "32-bit";
3841	case PGMSLAT_PAE: return "PAE";
3842	case PGMSLAT_AMD64: return "AMD64";
3843	default: return "Unknown";
3844	}
3845	}
3846	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3847
3848
3849	#ifdef VBOX_VMM_TARGET_X86
3850	/**
3851	* Notification from CPUM that the EFER.NXE bit has changed.
3852	*
3853	* @param pVCpu The cross context virtual CPU structure of the CPU for
3854	* which EFER changed.
3855	* @param fNxe The new NXE state.
3856	*/
3857	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3858	{
3859	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3860	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3861
3862	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3863	if (fNxe)
3864	{
3865	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3866	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3867	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3868	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3869	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3870	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3871	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3872	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3873	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3874	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3875	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3876
3877	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3878	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3879	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3880	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3881	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3882	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3883	}
3884	else
3885	{
3886	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3887	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3888	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3889	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3890	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3891	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3892	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3893	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3894	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3895	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3896	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3897
3898	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3899	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3900	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3901	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3902	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3903	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3904	}
3905	}
3906	#endif /* VBOX_VMM_TARGET_X86 */
3907
3908
3909	/**
3910	* Check if any pgm pool pages are marked dirty (not monitored)
3911	*
3912	* @returns bool locked/not locked
3913	* @param pVM The cross context VM structure.
3914	*/
3915	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3916	{
3917	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3918	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3919	#else
3920	RT_NOREF(pVM);
3921	return false;
3922	#endif
3923	}
3924
3925
3926	/**
3927	* Enable or disable large page usage
3928	*
3929	* @returns VBox status code.
3930	* @param pVM The cross context VM structure.
3931	* @param fUseLargePages Use/not use large pages
3932	*/
3933	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3934	{
3935	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3936
3937	pVM->pgm.s.fUseLargePages = fUseLargePages;
3938	return VINF_SUCCESS;
3939	}
3940
3941
3942	/**
3943	* Check if this VCPU currently owns the PGM lock.
3944	*
3945	* @returns bool owner/not owner
3946	* @param pVM The cross context VM structure.
3947	*/
3948	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3949	{
3950	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3951	}
3952
3953
3954	/**
3955	* Acquire the PGM lock.
3956	*
3957	* @returns VBox status code
3958	* @param pVM The cross context VM structure.
3959	* @param fVoid Set if the caller cannot handle failure returns.
3960	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3961	*/
3962	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3963	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3964	#else
3965	int pgmLock(PVMCC pVM, bool fVoid)
3966	#endif
3967	{
3968	#if defined(VBOX_STRICT)
3969	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3970	#else
3971	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3972	#endif
3973	if (RT_SUCCESS(rc))
3974	return rc;
3975	if (fVoid)
3976	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3977	else
3978	AssertRC(rc);
3979	return rc;
3980	}
3981
3982
3983	/**
3984	* Release the PGM lock.
3985	*
3986	* @param pVM The cross context VM structure.
3987	*/
3988	void pgmUnlock(PVMCC pVM)
3989	{
3990	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3991	pVM->pgm.s.cDeprecatedPageLocks = 0;
3992	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3993	if (rc == VINF_SEM_NESTED)
3994	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3995	}
3996
3997
3998	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3999
4000	/** Format handler for PGMPAGE.
4001	* @copydoc FNRTSTRFORMATTYPE */
4002	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
4003	const char pszType, void const pvValue,
4004	int cchWidth, int cchPrecision, unsigned fFlags,
4005	void *pvUser)
4006	{
4007	size_t cch;
4008	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
4009	if (RT_VALID_PTR(pPage))
4010	{
4011	char szTmp[64+80];
4012
4013	cch = 0;
4014
4015	/* The single char state stuff. */
4016	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
4017	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
4018
4019	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
4020	if (IS_PART_INCLUDED(5))
4021	{
4022	static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
4023	szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
4024	\| ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
4025	}
4026
4027	/* The type. */
4028	if (IS_PART_INCLUDED(4))
4029	{
4030	szTmp[cch++] = ':';
4031	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
4032	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
4033	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
4034	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
4035	}
4036
4037	/* The numbers. */
4038	if (IS_PART_INCLUDED(3))
4039	{
4040	szTmp[cch++] = ':';
4041	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
4042	}
4043
4044	if (IS_PART_INCLUDED(2))
4045	{
4046	szTmp[cch++] = ':';
4047	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
4048	}
4049
4050	if (IS_PART_INCLUDED(6))
4051	{
4052	szTmp[cch++] = ':';
4053	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
4054	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
4055	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
4056	}
4057	# undef IS_PART_INCLUDED
4058
4059	cch = pfnOutput(pvArgOutput, szTmp, cch);
4060	#if 0
4061	size_t cch2 = 0;
4062	szTmp[cch2++] = '(';
4063	cch2 += RTStrFormatNumber(&szTmp[cch2], (uintptr_t)pPage, 16, 18, 0, RTSTR_F_SPECIAL \| RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
4064	szTmp[cch2++] = ')';
4065	szTmp[cch2] = '\0';
4066	cch += pfnOutput(pvArgOutput, szTmp, cch2);
4067	#endif
4068	}
4069	else
4070	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
4071	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
4072	return cch;
4073	}
4074
4075
4076	/** Format handler for PGMRAMRANGE.
4077	* @copydoc FNRTSTRFORMATTYPE */
4078	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
4079	const char pszType, void const pvValue,
4080	int cchWidth, int cchPrecision, unsigned fFlags,
4081	void *pvUser)
4082	{
4083	size_t cch;
4084	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
4085	if (RT_VALID_PTR(pRam))
4086	{
4087	char szTmp[80];
4088	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
4089	cch = pfnOutput(pvArgOutput, szTmp, cch);
4090	}
4091	else
4092	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
4093	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
4094	return cch;
4095	}
4096
4097	/** Format type andlers to be registered/deregistered. */
4098	static const struct
4099	{
4100	char szType[24];
4101	PFNRTSTRFORMATTYPE pfnHandler;
4102	} g_aPgmFormatTypes[] =
4103	{
4104	{ "pgmpage", pgmFormatTypeHandlerPage },
4105	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
4106	};
4107
4108	#endif /* !IN_R0 \|\| LOG_ENABLED */
4109
4110	/**
4111	* Registers the global string format types.
4112	*
4113	* This should be called at module load time or in some other manner that ensure
4114	* that it's called exactly one time.
4115	*
4116	* @returns IPRT status code on RTStrFormatTypeRegister failure.
4117	*/
4118	VMMDECL(int) PGMRegisterStringFormatTypes(void)
4119	{
4120	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4121	int rc = VINF_SUCCESS;
4122	unsigned i;
4123	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4124	{
4125	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4126	# ifdef IN_RING0
4127	if (rc == VERR_ALREADY_EXISTS)
4128	{
4129	/* in case of cleanup failure in ring-0 */
4130	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4131	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4132	}
4133	# endif
4134	}
4135	if (RT_FAILURE(rc))
4136	while (i-- > 0)
4137	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4138
4139	return rc;
4140	#else
4141	return VINF_SUCCESS;
4142	#endif
4143	}
4144
4145
4146	/**
4147	* Deregisters the global string format types.
4148	*
4149	* This should be called at module unload time or in some other manner that
4150	* ensure that it's called exactly one time.
4151	*/
4152	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
4153	{
4154	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4155	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4156	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4157	#endif
4158	}
4159
4160	#ifdef VBOX_VMM_TARGET_X86
4161
4162	# ifdef VBOX_STRICT
4163	/**
4164	* Asserts that everything related to the guest CR3 is correctly shadowed.
4165	*
4166	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
4167	* and assert the correctness of the guest CR3 mapping before asserting that the
4168	* shadow page tables is in sync with the guest page tables.
4169	*
4170	* @returns Number of conflicts.
4171	* @param pVM The cross context VM structure.
4172	* @param pVCpu The cross context virtual CPU structure.
4173	* @param cr3 The current guest CR3 register value.
4174	* @param cr4 The current guest CR4 register value.
4175	*/
4176	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
4177	{
4178	AssertReturn(pVM->enmTarget == VMTARGET_X86, 0);
4179	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4180
4181	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
4182	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
4183	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
4184
4185	PGM_LOCK_VOID(pVM);
4186	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
4187	PGM_UNLOCK(pVM);
4188
4189	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4190	return cErrors;
4191	}
4192	# endif /* VBOX_STRICT */
4193
4194
4195	/**
4196	* Updates PGM's copy of the guest's EPT pointer.
4197	*
4198	* @param pVCpu The cross context virtual CPU structure.
4199	* @param uEptPtr The EPT pointer.
4200	*
4201	* @remarks This can be called as part of VM-entry so we might be in the midst of
4202	* switching to VMX non-root mode.
4203	*/
4204	VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
4205	{
4206	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4207	PGM_LOCK_VOID(pVM);
4208	pVCpu->pgm.s.uEptPtr = uEptPtr;
4209	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4210	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4211	PGM_UNLOCK(pVM);
4212	}
4213
4214	#endif /* VBOX_VMM_TARGET_X86 */
4215	#ifdef PGM_WITH_PAGE_ZEROING_DETECTION
4216	# ifndef VBOX_VMM_TARGET_X86
4217	# error "misconfig: PGM_WITH_PAGE_ZEROING_DETECTION not implemented for ARM guests"
4218	# endif
4219
4220	/**
4221	* Helper for checking whether XMM0 is zero, possibly retriving external state.
4222	*/
4223	static bool pgmHandlePageZeroingIsXmm0Zero(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4224	{
4225	if (pCtx->fExtrn & CPUMCTX_EXTRN_SSE_AVX)
4226	{
4227	int rc = CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_SSE_AVX);
4228	AssertRCReturn(rc, false);
4229	}
4230	return pCtx->XState.x87.aXMM[0].au64[0] == 0
4231	&& pCtx->XState.x87.aXMM[0].au64[1] == 0
4232	&& pCtx->XState.x87.aXMM[0].au64[2] == 0
4233	&& pCtx->XState.x87.aXMM[0].au64[3] == 0;
4234	}
4235
4236
4237	/**
4238	* Helper for comparing opcode bytes.
4239	*/
4240	static bool pgmHandlePageZeroingMatchOpcodes(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t const *pbOpcodes, uint32_t cbOpcodes)
4241	{
4242	uint8_t abTmp[64];
4243	AssertMsgReturn(cbOpcodes <= sizeof(abTmp), ("cbOpcodes=%#x\n", cbOpcodes), false);
4244	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abTmp, pCtx->rip + pCtx->cs.u64Base, cbOpcodes);
4245	if (RT_SUCCESS(rc))
4246	return memcmp(abTmp, pbOpcodes, cbOpcodes) == 0;
4247	return false;
4248	}
4249
4250
4251	/**
4252	* Called on faults on ZERO pages to check if the guest is trying to zero it.
4253	*
4254	* Since it's a waste of time to zero a ZERO page and it will cause an
4255	* unnecessary page allocation, we'd like to detect and avoid this.
4256	* If any known page zeroing code is detected, this function will update the CPU
4257	* state to pretend the page was zeroed by the code.
4258	*
4259	* @returns true if page zeroing code was detected and CPU state updated to skip
4260	* the code.
4261	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4262	* @param pCtx The guest register context.
4263	*/
4264	static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4265	{
4266	CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
4267
4268	/*
4269	* Sort by mode first.
4270	*/
4271	if (CPUMIsGuestInLongModeEx(pCtx))
4272	{
4273	if (CPUMIsGuestIn64BitCodeEx(pCtx))
4274	{
4275	/*
4276	* 64-bit code.
4277	*/
4278	Log9(("pgmHandlePageZeroingCode: not page zeroing - 64-bit\n"));
4279	}
4280	else if (pCtx->cs.Attr.n.u1DefBig)
4281	Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit lm\n"));
4282	else
4283	Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit lm\n"));
4284	}
4285	else if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
4286	{
4287	if (pCtx->cs.Attr.n.u1DefBig)
4288	{
4289	/*
4290	* 32-bit paged protected mode code.
4291	*/
4292	CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX
4293	\| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP \| CPUMCTX_EXTRN_RSI \| CPUMCTX_EXTRN_RDI
4294	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4295
4296	/* 1. Generic 'rep stosd' detection. */
4297	static uint8_t const s_abRepStosD[] = { 0xf3, 0xab };
4298	if ( pCtx->eax == 0
4299	&& pCtx->ecx == X86_PAGE_SIZE / 4
4300	&& !(pCtx->edi & X86_PAGE_OFFSET_MASK)
4301	&& pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abRepStosD, sizeof(s_abRepStosD)))
4302	{
4303	pCtx->ecx = 0;
4304	pCtx->edi += X86_PAGE_SIZE;
4305	Log9(("pgmHandlePageZeroingCode: REP STOSD: eip=%RX32 -> %RX32\n", pCtx->eip, pCtx->eip + sizeof(s_abRepStosD)));
4306	pCtx->eip += sizeof(s_abRepStosD);
4307	return true;
4308	}
4309
4310	/* 2. Windows 2000 sp4 KiXMMIZeroPageNoSave loop code: */
4311	static uint8_t const s_abW2kSp4XmmZero[] =
4312	{
4313	0x0f, 0x2b, 0x01,
4314	0x0f, 0x2b, 0x41, 0x10,
4315	0x0f, 0x2b, 0x41, 0x20,
4316	0x0f, 0x2b, 0x41, 0x30,
4317	0x83, 0xc1, 0x40,
4318	0x48,
4319	0x75, 0xeb,
4320	};
4321	if ( pCtx->eax == 64
4322	&& !(pCtx->ecx & X86_PAGE_OFFSET_MASK)
4323	&& pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abW2kSp4XmmZero, sizeof(s_abW2kSp4XmmZero))
4324	&& pgmHandlePageZeroingIsXmm0Zero(pVCpu, pCtx))
4325	{
4326	pCtx->eax = 1;
4327	pCtx->ecx += X86_PAGE_SIZE;
4328	Log9(("pgmHandlePageZeroingCode: w2k sp4 xmm: eip=%RX32 -> %RX32\n",
4329	pCtx->eip, pCtx->eip + sizeof(s_abW2kSp4XmmZero) - 3));
4330	pCtx->eip += sizeof(s_abW2kSp4XmmZero) - 3;
4331	return true;
4332	}
4333	Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit\n"));
4334	}
4335	else if (!pCtx->eflags.Bits.u1VM)
4336	Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit\n"));
4337	else
4338	Log9(("pgmHandlePageZeroingCode: not page zeroing - v86\n"));
4339	}
4340	return false;
4341	}
4342
4343	#endif /* PGM_WITH_PAGE_ZEROING_DETECTION */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 108132

Download in other formats: