PGMAll.cpp@ 108134

Last change on this file since 108134 was 108134, checked in by vboxsync, 12 days ago
VMM/PGM: Merge and deduplicate code targeting x86 & amd64 in PGM.cpp. Don't bother compiling pool stuff on arm and darwin.amd64. jiraref:VBP-1531
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 157.1 KB

Line
1	/* $Id: PGMAll.cpp 108134 2025-02-10 11:43:35Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#ifdef IN_RING0
35	# define VBOX_VMM_TARGET_X86
36	#endif
37	#include <VBox/vmm/pgm.h>
38	#include <VBox/vmm/cpum.h>
39	#include <VBox/vmm/selm.h>
40	#include <VBox/vmm/iem.h>
41	#include <VBox/vmm/iom.h>
42	#include <VBox/sup.h>
43	#include <VBox/vmm/mm.h>
44	#include <VBox/vmm/stam.h>
45	#include <VBox/vmm/trpm.h>
46	#include <VBox/vmm/em.h>
47	#include <VBox/vmm/hm.h>
48	#include <VBox/vmm/hm_vmx.h>
49	#include "PGMInternal.h"
50	#include <VBox/vmm/vmcc.h>
51	#include "PGMInline.h"
52	#include <iprt/assert.h>
53	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_X86)
54	# include <iprt/asm-amd64-x86.h>
55	#endif
56	#include <iprt/string.h>
57	#include <VBox/log.h>
58	#include <VBox/param.h>
59	#include <VBox/err.h>
60
61
62	/*********************************************************************************************************************************
63	* Internal Functions *
64	*********************************************************************************************************************************/
65	#ifdef VBOX_VMM_TARGET_X86
66	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
67	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
68	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
69	# endif
70	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
71	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
72	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
73	PPGMPTWALKGST pGstWalk);
74	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
75	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
76	PPGMPTWALKGST pGstWalkAll);
77	# endif
78	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
79	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
80	# ifdef PGM_WITH_PAGE_ZEROING_DETECTION
81	static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx);
82	# endif
83	#endif /* VBOX_VMM_TARGET_X86 */
84
85
86	#ifdef VBOX_VMM_TARGET_X86
87
88	/*
89	* Second level transation - EPT.
90	*/
91	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
92	# define PGM_SLAT_TYPE PGM_SLAT_TYPE_EPT
93	# include "PGMSlatDefs.h"
94	# include "PGMAllGstSlatEpt.cpp.h"
95	# undef PGM_SLAT_TYPE
96	# endif
97
98
99	/*
100	* Shadow - 32-bit mode
101	*/
102	# define PGM_SHW_TYPE PGM_TYPE_32BIT
103	# define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
104	# include "PGMAllShw.h"
105
106	/* Guest - real mode */
107	# define PGM_GST_TYPE PGM_TYPE_REAL
108	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
109	# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
110	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
111	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
112	# include "PGMGstDefs.h"
113	# include "PGMAllGst.h"
114	# include "PGMAllBth.h"
115	# undef BTH_PGMPOOLKIND_PT_FOR_PT
116	# undef BTH_PGMPOOLKIND_ROOT
117	# undef PGM_BTH_NAME
118	# undef PGM_GST_TYPE
119	# undef PGM_GST_NAME
120
121	/* Guest - protected mode */
122	# define PGM_GST_TYPE PGM_TYPE_PROT
123	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
124	# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
125	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
126	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
127	# include "PGMGstDefs.h"
128	# include "PGMAllGst.h"
129	# include "PGMAllBth.h"
130	# undef BTH_PGMPOOLKIND_PT_FOR_PT
131	# undef BTH_PGMPOOLKIND_ROOT
132	# undef PGM_BTH_NAME
133	# undef PGM_GST_TYPE
134	# undef PGM_GST_NAME
135
136	/* Guest - 32-bit mode */
137	# define PGM_GST_TYPE PGM_TYPE_32BIT
138	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
139	# define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
140	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
141	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
142	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
143	# include "PGMGstDefs.h"
144	# include "PGMAllGst.h"
145	# include "PGMAllBth.h"
146	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
147	# undef BTH_PGMPOOLKIND_PT_FOR_PT
148	# undef BTH_PGMPOOLKIND_ROOT
149	# undef PGM_BTH_NAME
150	# undef PGM_GST_TYPE
151	# undef PGM_GST_NAME
152
153	# undef PGM_SHW_TYPE
154	# undef PGM_SHW_NAME
155
156
157	/*
158	* Shadow - PAE mode
159	*/
160	# define PGM_SHW_TYPE PGM_TYPE_PAE
161	# define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
162	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
163	# include "PGMAllShw.h"
164
165	/* Guest - real mode */
166	# define PGM_GST_TYPE PGM_TYPE_REAL
167	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
168	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
169	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
170	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
171	# include "PGMGstDefs.h"
172	# include "PGMAllBth.h"
173	# undef BTH_PGMPOOLKIND_PT_FOR_PT
174	# undef BTH_PGMPOOLKIND_ROOT
175	# undef PGM_BTH_NAME
176	# undef PGM_GST_TYPE
177	# undef PGM_GST_NAME
178
179	/* Guest - protected mode */
180	# define PGM_GST_TYPE PGM_TYPE_PROT
181	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
182	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
183	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
184	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
185	# include "PGMGstDefs.h"
186	# include "PGMAllBth.h"
187	# undef BTH_PGMPOOLKIND_PT_FOR_PT
188	# undef BTH_PGMPOOLKIND_ROOT
189	# undef PGM_BTH_NAME
190	# undef PGM_GST_TYPE
191	# undef PGM_GST_NAME
192
193	/* Guest - 32-bit mode */
194	# define PGM_GST_TYPE PGM_TYPE_32BIT
195	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
196	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
197	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
198	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
199	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
200	# include "PGMGstDefs.h"
201	# include "PGMAllBth.h"
202	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
203	# undef BTH_PGMPOOLKIND_PT_FOR_PT
204	# undef BTH_PGMPOOLKIND_ROOT
205	# undef PGM_BTH_NAME
206	# undef PGM_GST_TYPE
207	# undef PGM_GST_NAME
208
209
210	/* Guest - PAE mode */
211	# define PGM_GST_TYPE PGM_TYPE_PAE
212	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
213	# define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
214	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
215	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
216	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
217	# include "PGMGstDefs.h"
218	# include "PGMAllGst.h"
219	# include "PGMAllBth.h"
220	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
221	# undef BTH_PGMPOOLKIND_PT_FOR_PT
222	# undef BTH_PGMPOOLKIND_ROOT
223	# undef PGM_BTH_NAME
224	# undef PGM_GST_TYPE
225	# undef PGM_GST_NAME
226
227	# undef PGM_SHW_TYPE
228	# undef PGM_SHW_NAME
229
230
231	/*
232	* Shadow - AMD64 mode
233	*/
234	# define PGM_SHW_TYPE PGM_TYPE_AMD64
235	# define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
236	# include "PGMAllShw.h"
237
238	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
239	/** @todo retire this hack. */
240	# define PGM_GST_TYPE PGM_TYPE_PROT
241	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
242	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
243	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
244	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
245	# include "PGMGstDefs.h"
246	# include "PGMAllBth.h"
247	# undef BTH_PGMPOOLKIND_PT_FOR_PT
248	# undef BTH_PGMPOOLKIND_ROOT
249	# undef PGM_BTH_NAME
250	# undef PGM_GST_TYPE
251	# undef PGM_GST_NAME
252
253	# ifdef VBOX_WITH_64_BITS_GUESTS
254	/* Guest - AMD64 mode */
255	# define PGM_GST_TYPE PGM_TYPE_AMD64
256	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
257	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
258	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
259	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
260	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
261	# include "PGMGstDefs.h"
262	# include "PGMAllGst.h"
263	# include "PGMAllBth.h"
264	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
265	# undef BTH_PGMPOOLKIND_PT_FOR_PT
266	# undef BTH_PGMPOOLKIND_ROOT
267	# undef PGM_BTH_NAME
268	# undef PGM_GST_TYPE
269	# undef PGM_GST_NAME
270	# endif /* VBOX_WITH_64_BITS_GUESTS */
271
272	# undef PGM_SHW_TYPE
273	# undef PGM_SHW_NAME
274
275
276	/*
277	* Shadow - 32-bit nested paging mode.
278	*/
279	# define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
280	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
281	# include "PGMAllShw.h"
282
283	/* Guest - real mode */
284	# define PGM_GST_TYPE PGM_TYPE_REAL
285	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
286	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
287	# include "PGMGstDefs.h"
288	# include "PGMAllBth.h"
289	# undef PGM_BTH_NAME
290	# undef PGM_GST_TYPE
291	# undef PGM_GST_NAME
292
293	/* Guest - protected mode */
294	# define PGM_GST_TYPE PGM_TYPE_PROT
295	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
296	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
297	# include "PGMGstDefs.h"
298	# include "PGMAllBth.h"
299	# undef PGM_BTH_NAME
300	# undef PGM_GST_TYPE
301	# undef PGM_GST_NAME
302
303	/* Guest - 32-bit mode */
304	# define PGM_GST_TYPE PGM_TYPE_32BIT
305	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
306	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
307	# include "PGMGstDefs.h"
308	# include "PGMAllBth.h"
309	# undef PGM_BTH_NAME
310	# undef PGM_GST_TYPE
311	# undef PGM_GST_NAME
312
313	/* Guest - PAE mode */
314	# define PGM_GST_TYPE PGM_TYPE_PAE
315	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
316	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
317	# include "PGMGstDefs.h"
318	# include "PGMAllBth.h"
319	# undef PGM_BTH_NAME
320	# undef PGM_GST_TYPE
321	# undef PGM_GST_NAME
322
323	# ifdef VBOX_WITH_64_BITS_GUESTS
324	/* Guest - AMD64 mode */
325	# define PGM_GST_TYPE PGM_TYPE_AMD64
326	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
327	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
328	# include "PGMGstDefs.h"
329	# include "PGMAllBth.h"
330	# undef PGM_BTH_NAME
331	# undef PGM_GST_TYPE
332	# undef PGM_GST_NAME
333	# endif /* VBOX_WITH_64_BITS_GUESTS */
334
335	# undef PGM_SHW_TYPE
336	# undef PGM_SHW_NAME
337
338
339	/*
340	* Shadow - PAE nested paging mode.
341	*/
342	# define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
343	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
344	# include "PGMAllShw.h"
345
346	/* Guest - real mode */
347	# define PGM_GST_TYPE PGM_TYPE_REAL
348	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
349	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
350	# include "PGMGstDefs.h"
351	# include "PGMAllBth.h"
352	# undef PGM_BTH_NAME
353	# undef PGM_GST_TYPE
354	# undef PGM_GST_NAME
355
356	/* Guest - protected mode */
357	# define PGM_GST_TYPE PGM_TYPE_PROT
358	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
359	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
360	# include "PGMGstDefs.h"
361	# include "PGMAllBth.h"
362	# undef PGM_BTH_NAME
363	# undef PGM_GST_TYPE
364	# undef PGM_GST_NAME
365
366	/* Guest - 32-bit mode */
367	# define PGM_GST_TYPE PGM_TYPE_32BIT
368	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
369	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
370	# include "PGMGstDefs.h"
371	# include "PGMAllBth.h"
372	# undef PGM_BTH_NAME
373	# undef PGM_GST_TYPE
374	# undef PGM_GST_NAME
375
376	/* Guest - PAE mode */
377	# define PGM_GST_TYPE PGM_TYPE_PAE
378	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
379	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
380	# include "PGMGstDefs.h"
381	# include "PGMAllBth.h"
382	# undef PGM_BTH_NAME
383	# undef PGM_GST_TYPE
384	# undef PGM_GST_NAME
385
386	# ifdef VBOX_WITH_64_BITS_GUESTS
387	/* Guest - AMD64 mode */
388	# define PGM_GST_TYPE PGM_TYPE_AMD64
389	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
390	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
391	# include "PGMGstDefs.h"
392	# include "PGMAllBth.h"
393	# undef PGM_BTH_NAME
394	# undef PGM_GST_TYPE
395	# undef PGM_GST_NAME
396	# endif /* VBOX_WITH_64_BITS_GUESTS */
397
398	# undef PGM_SHW_TYPE
399	# undef PGM_SHW_NAME
400
401
402	/*
403	* Shadow - AMD64 nested paging mode.
404	*/
405	# define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
406	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
407	# include "PGMAllShw.h"
408
409	/* Guest - real mode */
410	# define PGM_GST_TYPE PGM_TYPE_REAL
411	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
412	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
413	# include "PGMGstDefs.h"
414	# include "PGMAllBth.h"
415	# undef PGM_BTH_NAME
416	# undef PGM_GST_TYPE
417	# undef PGM_GST_NAME
418
419	/* Guest - protected mode */
420	# define PGM_GST_TYPE PGM_TYPE_PROT
421	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
422	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
423	# include "PGMGstDefs.h"
424	# include "PGMAllBth.h"
425	# undef PGM_BTH_NAME
426	# undef PGM_GST_TYPE
427	# undef PGM_GST_NAME
428
429	/* Guest - 32-bit mode */
430	# define PGM_GST_TYPE PGM_TYPE_32BIT
431	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
432	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
433	# include "PGMGstDefs.h"
434	# include "PGMAllBth.h"
435	# undef PGM_BTH_NAME
436	# undef PGM_GST_TYPE
437	# undef PGM_GST_NAME
438
439	/* Guest - PAE mode */
440	# define PGM_GST_TYPE PGM_TYPE_PAE
441	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
442	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
443	# include "PGMGstDefs.h"
444	# include "PGMAllBth.h"
445	# undef PGM_BTH_NAME
446	# undef PGM_GST_TYPE
447	# undef PGM_GST_NAME
448
449	# ifdef VBOX_WITH_64_BITS_GUESTS
450	/* Guest - AMD64 mode */
451	# define PGM_GST_TYPE PGM_TYPE_AMD64
452	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
453	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
454	# include "PGMGstDefs.h"
455	# include "PGMAllBth.h"
456	# undef PGM_BTH_NAME
457	# undef PGM_GST_TYPE
458	# undef PGM_GST_NAME
459	# endif /* VBOX_WITH_64_BITS_GUESTS */
460
461	# undef PGM_SHW_TYPE
462	# undef PGM_SHW_NAME
463
464
465	/*
466	* Shadow - EPT.
467	*/
468	# define PGM_SHW_TYPE PGM_TYPE_EPT
469	# define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
470	# include "PGMAllShw.h"
471
472	/* Guest - real mode */
473	# define PGM_GST_TYPE PGM_TYPE_REAL
474	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
475	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
476	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
477	# include "PGMGstDefs.h"
478	# include "PGMAllBth.h"
479	# undef BTH_PGMPOOLKIND_PT_FOR_PT
480	# undef PGM_BTH_NAME
481	# undef PGM_GST_TYPE
482	# undef PGM_GST_NAME
483
484	/* Guest - protected mode */
485	# define PGM_GST_TYPE PGM_TYPE_PROT
486	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
487	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
488	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
489	# include "PGMGstDefs.h"
490	# include "PGMAllBth.h"
491	# undef BTH_PGMPOOLKIND_PT_FOR_PT
492	# undef PGM_BTH_NAME
493	# undef PGM_GST_TYPE
494	# undef PGM_GST_NAME
495
496	/* Guest - 32-bit mode */
497	# define PGM_GST_TYPE PGM_TYPE_32BIT
498	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
499	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
500	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
501	# include "PGMGstDefs.h"
502	# include "PGMAllBth.h"
503	# undef BTH_PGMPOOLKIND_PT_FOR_PT
504	# undef PGM_BTH_NAME
505	# undef PGM_GST_TYPE
506	# undef PGM_GST_NAME
507
508	/* Guest - PAE mode */
509	# define PGM_GST_TYPE PGM_TYPE_PAE
510	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
511	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
512	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
513	# include "PGMGstDefs.h"
514	# include "PGMAllBth.h"
515	# undef BTH_PGMPOOLKIND_PT_FOR_PT
516	# undef PGM_BTH_NAME
517	# undef PGM_GST_TYPE
518	# undef PGM_GST_NAME
519
520	# ifdef VBOX_WITH_64_BITS_GUESTS
521	/* Guest - AMD64 mode */
522	# define PGM_GST_TYPE PGM_TYPE_AMD64
523	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
524	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
525	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
526	# include "PGMGstDefs.h"
527	# include "PGMAllBth.h"
528	# undef BTH_PGMPOOLKIND_PT_FOR_PT
529	# undef PGM_BTH_NAME
530	# undef PGM_GST_TYPE
531	# undef PGM_GST_NAME
532	# endif /* VBOX_WITH_64_BITS_GUESTS */
533
534	# undef PGM_SHW_TYPE
535	# undef PGM_SHW_NAME
536
537
538	/*
539	* Shadow - NEM / None.
540	*/
541	# define PGM_SHW_TYPE PGM_TYPE_NONE
542	# define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
543	# include "PGMAllShw.h"
544
545	/* Guest - real mode */
546	# define PGM_GST_TYPE PGM_TYPE_REAL
547	# define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
548	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
549	# include "PGMGstDefs.h"
550	# include "PGMAllBth.h"
551	# undef PGM_BTH_NAME
552	# undef PGM_GST_TYPE
553	# undef PGM_GST_NAME
554
555	/* Guest - protected mode */
556	# define PGM_GST_TYPE PGM_TYPE_PROT
557	# define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
558	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
559	# include "PGMGstDefs.h"
560	# include "PGMAllBth.h"
561	# undef PGM_BTH_NAME
562	# undef PGM_GST_TYPE
563	# undef PGM_GST_NAME
564
565	/* Guest - 32-bit mode */
566	# define PGM_GST_TYPE PGM_TYPE_32BIT
567	# define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
568	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
569	# include "PGMGstDefs.h"
570	# include "PGMAllBth.h"
571	# undef PGM_BTH_NAME
572	# undef PGM_GST_TYPE
573	# undef PGM_GST_NAME
574
575	/* Guest - PAE mode */
576	# define PGM_GST_TYPE PGM_TYPE_PAE
577	# define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
578	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
579	# include "PGMGstDefs.h"
580	# include "PGMAllBth.h"
581	# undef PGM_BTH_NAME
582	# undef PGM_GST_TYPE
583	# undef PGM_GST_NAME
584
585	# ifdef VBOX_WITH_64_BITS_GUESTS
586	/* Guest - AMD64 mode */
587	# define PGM_GST_TYPE PGM_TYPE_AMD64
588	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
589	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
590	# include "PGMGstDefs.h"
591	# include "PGMAllBth.h"
592	# undef PGM_BTH_NAME
593	# undef PGM_GST_TYPE
594	# undef PGM_GST_NAME
595	# endif /* VBOX_WITH_64_BITS_GUESTS */
596
597	# undef PGM_SHW_TYPE
598	# undef PGM_SHW_NAME
599
600
601
602	/**
603	* Guest mode data array.
604	*/
605	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
606	{
607	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
608	{
609	PGM_TYPE_REAL,
610	PGM_GST_NAME_REAL(GetPage),
611	PGM_GST_NAME_REAL(QueryPageFast),
612	PGM_GST_NAME_REAL(ModifyPage),
613	PGM_GST_NAME_REAL(Enter),
614	PGM_GST_NAME_REAL(Exit),
615	# ifdef IN_RING3
616	PGM_GST_NAME_REAL(Relocate),
617	# endif
618	},
619	{
620	PGM_TYPE_PROT,
621	PGM_GST_NAME_PROT(GetPage),
622	PGM_GST_NAME_PROT(QueryPageFast),
623	PGM_GST_NAME_PROT(ModifyPage),
624	PGM_GST_NAME_PROT(Enter),
625	PGM_GST_NAME_PROT(Exit),
626	# ifdef IN_RING3
627	PGM_GST_NAME_PROT(Relocate),
628	# endif
629	},
630	{
631	PGM_TYPE_32BIT,
632	PGM_GST_NAME_32BIT(GetPage),
633	PGM_GST_NAME_32BIT(QueryPageFast),
634	PGM_GST_NAME_32BIT(ModifyPage),
635	PGM_GST_NAME_32BIT(Enter),
636	PGM_GST_NAME_32BIT(Exit),
637	# ifdef IN_RING3
638	PGM_GST_NAME_32BIT(Relocate),
639	# endif
640	},
641	{
642	PGM_TYPE_PAE,
643	PGM_GST_NAME_PAE(GetPage),
644	PGM_GST_NAME_PAE(QueryPageFast),
645	PGM_GST_NAME_PAE(ModifyPage),
646	PGM_GST_NAME_PAE(Enter),
647	PGM_GST_NAME_PAE(Exit),
648	# ifdef IN_RING3
649	PGM_GST_NAME_PAE(Relocate),
650	# endif
651	},
652	# ifdef VBOX_WITH_64_BITS_GUESTS
653	{
654	PGM_TYPE_AMD64,
655	PGM_GST_NAME_AMD64(GetPage),
656	PGM_GST_NAME_AMD64(QueryPageFast),
657	PGM_GST_NAME_AMD64(ModifyPage),
658	PGM_GST_NAME_AMD64(Enter),
659	PGM_GST_NAME_AMD64(Exit),
660	# ifdef IN_RING3
661	PGM_GST_NAME_AMD64(Relocate),
662	# endif
663	},
664	# endif
665	};
666
667
668	/**
669	* The shadow mode data array.
670	*/
671	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
672	{
673	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
674	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
675	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
676	{
677	PGM_TYPE_32BIT,
678	PGM_SHW_NAME_32BIT(GetPage),
679	PGM_SHW_NAME_32BIT(ModifyPage),
680	PGM_SHW_NAME_32BIT(Enter),
681	PGM_SHW_NAME_32BIT(Exit),
682	# ifdef IN_RING3
683	PGM_SHW_NAME_32BIT(Relocate),
684	# endif
685	},
686	{
687	PGM_TYPE_PAE,
688	PGM_SHW_NAME_PAE(GetPage),
689	PGM_SHW_NAME_PAE(ModifyPage),
690	PGM_SHW_NAME_PAE(Enter),
691	PGM_SHW_NAME_PAE(Exit),
692	# ifdef IN_RING3
693	PGM_SHW_NAME_PAE(Relocate),
694	# endif
695	},
696	{
697	PGM_TYPE_AMD64,
698	PGM_SHW_NAME_AMD64(GetPage),
699	PGM_SHW_NAME_AMD64(ModifyPage),
700	PGM_SHW_NAME_AMD64(Enter),
701	PGM_SHW_NAME_AMD64(Exit),
702	# ifdef IN_RING3
703	PGM_SHW_NAME_AMD64(Relocate),
704	# endif
705	},
706	{
707	PGM_TYPE_NESTED_32BIT,
708	PGM_SHW_NAME_NESTED_32BIT(GetPage),
709	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
710	PGM_SHW_NAME_NESTED_32BIT(Enter),
711	PGM_SHW_NAME_NESTED_32BIT(Exit),
712	# ifdef IN_RING3
713	PGM_SHW_NAME_NESTED_32BIT(Relocate),
714	# endif
715	},
716	{
717	PGM_TYPE_NESTED_PAE,
718	PGM_SHW_NAME_NESTED_PAE(GetPage),
719	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
720	PGM_SHW_NAME_NESTED_PAE(Enter),
721	PGM_SHW_NAME_NESTED_PAE(Exit),
722	# ifdef IN_RING3
723	PGM_SHW_NAME_NESTED_PAE(Relocate),
724	# endif
725	},
726	{
727	PGM_TYPE_NESTED_AMD64,
728	PGM_SHW_NAME_NESTED_AMD64(GetPage),
729	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
730	PGM_SHW_NAME_NESTED_AMD64(Enter),
731	PGM_SHW_NAME_NESTED_AMD64(Exit),
732	# ifdef IN_RING3
733	PGM_SHW_NAME_NESTED_AMD64(Relocate),
734	# endif
735	},
736	{
737	PGM_TYPE_EPT,
738	PGM_SHW_NAME_EPT(GetPage),
739	PGM_SHW_NAME_EPT(ModifyPage),
740	PGM_SHW_NAME_EPT(Enter),
741	PGM_SHW_NAME_EPT(Exit),
742	# ifdef IN_RING3
743	PGM_SHW_NAME_EPT(Relocate),
744	# endif
745	},
746	{
747	PGM_TYPE_NONE,
748	PGM_SHW_NAME_NONE(GetPage),
749	PGM_SHW_NAME_NONE(ModifyPage),
750	PGM_SHW_NAME_NONE(Enter),
751	PGM_SHW_NAME_NONE(Exit),
752	# ifdef IN_RING3
753	PGM_SHW_NAME_NONE(Relocate),
754	# endif
755	},
756	};
757
758
759	/**
760	* The guest+shadow mode data array.
761	*/
762	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
763	{
764	# if !defined(IN_RING3) && !defined(VBOX_STRICT)
765	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
766	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
767	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
768
769	# elif !defined(IN_RING3) && defined(VBOX_STRICT)
770	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
771	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
772	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
773
774	# elif defined(IN_RING3) && !defined(VBOX_STRICT)
775	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
776	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
777	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
778
779	# elif defined(IN_RING3) && defined(VBOX_STRICT)
780	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
781	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
782	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
783
784	# else
785	# error "Misconfig."
786	# endif
787
788	/* 32-bit shadow paging mode: */
789	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
790	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
791	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
792	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
793	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
796	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
797	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
798	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
799	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
800
801	/* PAE shadow paging mode: */
802	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
803	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
805	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
806	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
807	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
808	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
810	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
811	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
812	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
813
814	/* AMD64 shadow paging mode: */
815	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
816	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
817	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
818	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
819	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
820	# ifdef VBOX_WITH_64_BITS_GUESTS
821	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
822	# else
823	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
824	# endif
825	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
827	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
828	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
829	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
830
831	/* 32-bit nested paging mode: */
832	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
833	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
834	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
835	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
837	# ifdef VBOX_WITH_64_BITS_GUESTS
838	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
839	# else
840	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
841	# endif
842	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
844	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
845	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
846	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
847
848	/* PAE nested paging mode: */
849	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
850	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
851	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
852	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
854	# ifdef VBOX_WITH_64_BITS_GUESTS
855	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
856	# else
857	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
858	# endif
859	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
861	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
862	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
863	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
864
865	/* AMD64 nested paging mode: */
866	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
867	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
868	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
869	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
870	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
871	# ifdef VBOX_WITH_64_BITS_GUESTS
872	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
873	# else
874	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
875	# endif
876	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
877	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
878	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
879	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
880	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
881
882	/* EPT nested paging mode: */
883	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
884	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
885	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
886	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
887	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
888	# ifdef VBOX_WITH_64_BITS_GUESTS
889	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
890	# else
891	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
892	# endif
893	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
894	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
895	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
896	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
897	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
898
899	/* NONE / NEM: */
900	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
901	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
902	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
903	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
904	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
905	# ifdef VBOX_WITH_64_BITS_GUESTS
906	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
907	# else
908	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
909	# endif
910	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
911	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
912	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
913	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
914	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
915
916
917	# undef PGMMODEDATABTH_ENTRY
918	# undef PGMMODEDATABTH_NULL_ENTRY
919	};
920
921
922	/** Mask array used by pgmGetCr3MaskForMode.
923	* X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
924	static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
925	{
926	/* [PGMMODE_INVALID] = */ X86_CR3_AMD64_PAGE_MASK,
927	/* [PGMMODE_REAL] = */ X86_CR3_AMD64_PAGE_MASK,
928	/* [PGMMODE_PROTECTED] = */ X86_CR3_AMD64_PAGE_MASK,
929	/* [PGMMODE_32_BIT] = */ X86_CR3_PAGE_MASK,
930	/* [PGMMODE_PAE] = */ X86_CR3_PAE_PAGE_MASK,
931	/* [PGMMODE_PAE_NX] = */ X86_CR3_PAE_PAGE_MASK,
932	/* [PGMMODE_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
933	/* [PGMMODE_AMD64_NX] = */ X86_CR3_AMD64_PAGE_MASK,
934	/* [PGMMODE_NESTED_32BIT = */ X86_CR3_PAGE_MASK,
935	/* [PGMMODE_NESTED_PAE] = */ X86_CR3_PAE_PAGE_MASK,
936	/* [PGMMODE_NESTED_AMD64] = */ X86_CR3_AMD64_PAGE_MASK,
937	/* [PGMMODE_EPT] = */ X86_CR3_EPT_PAGE_MASK,
938	/* [12] = */ 0,
939	/* [13] = */ 0,
940	/* [14] = */ 0,
941	/* [15] = */ 0,
942	/* [16] = */ 0,
943	/* [17] = */ 0,
944	/* [18] = */ 0,
945	/* [19] = */ 0,
946	/* [20] = */ 0,
947	/* [21] = */ 0,
948	/* [22] = */ 0,
949	/* [23] = */ 0,
950	/* [24] = */ 0,
951	/* [25] = */ 0,
952	/* [26] = */ 0,
953	/* [27] = */ 0,
954	/* [28] = */ 0,
955	/* [29] = */ 0,
956	/* [30] = */ 0,
957	/* [31] = */ 0,
958	/* [PGMMODE_NONE] = */ X86_CR3_AMD64_PAGE_MASK,
959	};
960	AssertCompile(PGMMODE_NONE == 32);
961
962
963	#elif defined(VBOX_VMM_TARGET_ARMV8)
964
965	/** @todo Proper ARMv8 guest support for PGM */
966	# include "PGMAllGst-armv8.h"
967
968	#else
969	# error "port me"
970	#endif
971
972
973	#ifdef VBOX_VMM_TARGET_X86
974
975	/**
976	* Gets the physical address mask for CR3 in the given paging mode.
977	*
978	* The mask is for eliminating flags and other stuff in CR3/EPTP when
979	* extracting the physical address. It is not for validating whether there are
980	* reserved bits set. PGM ASSUMES that whoever loaded the CR3 value and passed
981	* it to PGM checked for reserved bits, including reserved physical address
982	* bits.
983	*
984	* @returns The CR3 mask.
985	* @param enmMode The paging mode.
986	* @param enmSlatMode The second-level address translation mode.
987	*/
988	DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
989	{
990	if (enmSlatMode == PGMSLAT_DIRECT)
991	{
992	Assert(enmMode != PGMMODE_EPT);
993	return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
994	}
995	Assert(enmSlatMode == PGMSLAT_EPT);
996	return X86_CR3_EPT_PAGE_MASK;
997	}
998
999
1000	/**
1001	* Gets the masked CR3 value according to the current guest paging mode.
1002	*
1003	* See disclaimer in pgmGetCr3MaskForMode.
1004	*
1005	* @returns The masked PGM CR3 value.
1006	* @param pVCpu The cross context virtual CPU structure.
1007	* @param uCr3 The raw guest CR3 value.
1008	*/
1009	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
1010	{
1011	uint64_t const fCr3Mask = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
1012	RTGCPHYS GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
1013	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
1014	return GCPhysCR3;
1015	}
1016
1017
1018	# ifdef IN_RING0
1019	/**
1020	* \#PF Handler.
1021	*
1022	* @returns VBox status code (appropriate for trap handling and GC return).
1023	* @param pVCpu The cross context virtual CPU structure.
1024	* @param uErr The trap error code.
1025	* @param pCtx Pointer to the register context for the CPU.
1026	* @param pvFault The fault address.
1027	*/
1028	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
1029	{
1030	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1031
1032	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
1033	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
1034	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
1035
1036
1037	# ifdef VBOX_WITH_STATISTICS
1038	/*
1039	* Error code stats.
1040	*/
1041	if (uErr & X86_TRAP_PF_US)
1042	{
1043	if (!(uErr & X86_TRAP_PF_P))
1044	{
1045	if (uErr & X86_TRAP_PF_RW)
1046	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
1047	else
1048	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
1049	}
1050	else if (uErr & X86_TRAP_PF_RW)
1051	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
1052	else if (uErr & X86_TRAP_PF_RSVD)
1053	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
1054	else if (uErr & X86_TRAP_PF_ID)
1055	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
1056	else
1057	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
1058	}
1059	else
1060	{ /* Supervisor */
1061	if (!(uErr & X86_TRAP_PF_P))
1062	{
1063	if (uErr & X86_TRAP_PF_RW)
1064	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
1065	else
1066	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
1067	}
1068	else if (uErr & X86_TRAP_PF_RW)
1069	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
1070	else if (uErr & X86_TRAP_PF_ID)
1071	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
1072	else if (uErr & X86_TRAP_PF_RSVD)
1073	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
1074	}
1075	# endif /* VBOX_WITH_STATISTICS */
1076
1077	/*
1078	* Call the worker.
1079	*/
1080	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1081	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1082	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
1083	bool fLockTaken = false;
1084	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
1085	if (fLockTaken)
1086	{
1087	PGM_LOCK_ASSERT_OWNER(pVM);
1088	PGM_UNLOCK(pVM);
1089	}
1090	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
1091
1092	/*
1093	* Return code tweaks.
1094	*/
1095	if (rc != VINF_SUCCESS)
1096	{
1097	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
1098	rc = VINF_SUCCESS;
1099
1100	/* Note: hack alert for difficult to reproduce problem. */
1101	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
1102	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
1103	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
1104	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
1105	{
1106	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
1107	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
1108	rc = VINF_SUCCESS;
1109	}
1110	}
1111
1112	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
1113	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
1114	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
1115	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
1116	return rc;
1117	}
1118	# endif /* IN_RING0 */
1119
1120
1121	/**
1122	* Prefetch a page
1123	*
1124	* Typically used to sync commonly used pages before entering raw mode
1125	* after a CR3 reload.
1126	*
1127	* @returns VBox status code suitable for scheduling.
1128	* @retval VINF_SUCCESS on success.
1129	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
1130	* @param pVCpu The cross context virtual CPU structure.
1131	* @param GCPtrPage Page to invalidate.
1132	*/
1133	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1134	{
1135	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1136
1137	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1138	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1139	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1140	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1141
1142	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
1143	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1144	return rc;
1145	}
1146
1147
1148	/**
1149	* Emulation of the invlpg instruction (HC only actually).
1150	*
1151	* @returns Strict VBox status code, special care required.
1152	* @retval VINF_PGM_SYNC_CR3 - handled.
1153	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1154	*
1155	* @param pVCpu The cross context virtual CPU structure.
1156	* @param GCPtrPage Page to invalidate.
1157	*
1158	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1159	* safe, but there could be edge cases!
1160	*
1161	* @todo Flush page or page directory only if necessary!
1162	* @todo VBOXSTRICTRC
1163	*/
1164	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1165	{
1166	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1167	int rc;
1168	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1169
1170	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1171
1172	/*
1173	* Call paging mode specific worker.
1174	*/
1175	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1176	PGM_LOCK_VOID(pVM);
1177
1178	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1179	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1180	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1181	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1182
1183	PGM_UNLOCK(pVM);
1184	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1185
1186	/* Ignore all irrelevant error codes. */
1187	if ( rc == VERR_PAGE_NOT_PRESENT
1188	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1189	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1190	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1191	rc = VINF_SUCCESS;
1192
1193	return rc;
1194	}
1195
1196
1197	/**
1198	* Executes an instruction using the interpreter.
1199	*
1200	* @returns VBox status code (appropriate for trap handling and GC return).
1201	* @param pVCpu The cross context virtual CPU structure.
1202	* @param pvFault Fault address.
1203	*/
1204	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
1205	{
1206	RT_NOREF(pvFault);
1207	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
1208	if (rc == VERR_EM_INTERPRETER)
1209	rc = VINF_EM_RAW_EMULATE_INSTR;
1210	if (rc != VINF_SUCCESS)
1211	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1212	return rc;
1213	}
1214
1215
1216	/**
1217	* Gets effective page information (from the VMM page directory).
1218	*
1219	* @returns VBox status code.
1220	* @param pVCpu The cross context virtual CPU structure.
1221	* @param GCPtr Guest Context virtual address of the page.
1222	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1223	* @param pHCPhys Where to store the HC physical address of the page.
1224	* This is page aligned.
1225	* @remark You should use PGMMapGetPage() for pages in a mapping.
1226	*/
1227	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1228	{
1229	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1230	PGM_LOCK_VOID(pVM);
1231
1232	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1233	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1234	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1235	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1236
1237	PGM_UNLOCK(pVM);
1238	return rc;
1239	}
1240
1241
1242	/**
1243	* Modify page flags for a range of pages in the shadow context.
1244	*
1245	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1246	*
1247	* @returns VBox status code.
1248	* @param pVCpu The cross context virtual CPU structure.
1249	* @param GCPtr Virtual address of the first page in the range.
1250	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1251	* @param fMask The AND mask - page flags X86_PTE_*.
1252	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1253	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1254	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1255	*/
1256	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1257	{
1258	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1259	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1260
1261	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
1262
1263	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1264	PGM_LOCK_VOID(pVM);
1265
1266	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1267	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1268	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1269	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
1270
1271	PGM_UNLOCK(pVM);
1272	return rc;
1273	}
1274
1275
1276	/**
1277	* Changing the page flags for a single page in the shadow page tables so as to
1278	* make it read-only.
1279	*
1280	* @returns VBox status code.
1281	* @param pVCpu The cross context virtual CPU structure.
1282	* @param GCPtr Virtual address of the first page in the range.
1283	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1284	*/
1285	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1286	{
1287	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1288	}
1289
1290
1291	/**
1292	* Changing the page flags for a single page in the shadow page tables so as to
1293	* make it writable.
1294	*
1295	* The call must know with 101% certainty that the guest page tables maps this
1296	* as writable too. This function will deal shared, zero and write monitored
1297	* pages.
1298	*
1299	* @returns VBox status code.
1300	* @param pVCpu The cross context virtual CPU structure.
1301	* @param GCPtr Virtual address of the first page in the range.
1302	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1303	*/
1304	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1305	{
1306	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1307	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1308	return VINF_SUCCESS;
1309	}
1310
1311
1312	/**
1313	* Changing the page flags for a single page in the shadow page tables so as to
1314	* make it not present.
1315	*
1316	* @returns VBox status code.
1317	* @param pVCpu The cross context virtual CPU structure.
1318	* @param GCPtr Virtual address of the first page in the range.
1319	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1320	*/
1321	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1322	{
1323	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1324	}
1325
1326	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
1327
1328	/**
1329	* Changing the page flags for a single page in the shadow page tables so as to
1330	* make it supervisor and writable.
1331	*
1332	* This if for dealing with CR0.WP=0 and readonly user pages.
1333	*
1334	* @returns VBox status code.
1335	* @param pVCpu The cross context virtual CPU structure.
1336	* @param GCPtr Virtual address of the first page in the range.
1337	* @param fBigPage Whether or not this is a big page. If it is, we have to
1338	* change the shadow PDE as well. If it isn't, the caller
1339	* has checked that the shadow PDE doesn't need changing.
1340	* We ASSUME 4KB pages backing the big page here!
1341	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1342	*/
1343	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1344	{
1345	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1346	if (rc == VINF_SUCCESS && fBigPage)
1347	{
1348	/* this is a bit ugly... */
1349	switch (pVCpu->pgm.s.enmShadowMode)
1350	{
1351	case PGMMODE_32_BIT:
1352	{
1353	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1354	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1355	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1356	pPde->u \|= X86_PDE_RW;
1357	Log(("-> PDE=%#llx (32)\n", pPde->u));
1358	break;
1359	}
1360	case PGMMODE_PAE:
1361	case PGMMODE_PAE_NX:
1362	{
1363	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1364	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1365	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1366	pPde->u \|= X86_PDE_RW;
1367	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1368	break;
1369	}
1370	default:
1371	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1372	}
1373	}
1374	return rc;
1375	}
1376
1377
1378	/**
1379	* Gets the shadow page directory for the specified address, PAE.
1380	*
1381	* @returns Pointer to the shadow PD.
1382	* @param pVCpu The cross context virtual CPU structure.
1383	* @param GCPtr The address.
1384	* @param uGstPdpe Guest PDPT entry. Valid.
1385	* @param ppPD Receives address of page directory
1386	*/
1387	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1388	{
1389	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1390	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1391	PPGMPOOLPAGE pShwPage;
1392	int rc;
1393	PGM_LOCK_ASSERT_OWNER(pVM);
1394
1395
1396	/* Allocate page directory if not present. */
1397	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1398	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1399	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1400	X86PGPAEUINT const uPdpe = pPdpe->u;
1401	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1402	{
1403	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1404	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1405	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1406
1407	pgmPoolCacheUsed(pPool, pShwPage);
1408
1409	/* Update the entry if necessary. */
1410	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1411	if (uPdpeNew == uPdpe)
1412	{ /* likely */ }
1413	else
1414	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1415	}
1416	else
1417	{
1418	RTGCPTR64 GCPdPt;
1419	PGMPOOLKIND enmKind;
1420	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1421	{
1422	/* AMD-V nested paging or real/protected mode without paging. */
1423	GCPdPt = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
1424	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1425	}
1426	else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1427	{
1428	if (uGstPdpe & X86_PDPE_P)
1429	{
1430	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1431	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1432	}
1433	else
1434	{
1435	/* PD not present; guest must reload CR3 to change it.
1436	* No need to monitor anything in this case. */
1437	/** @todo r=bird: WTF is hit?!? */
1438	/Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? /
1439	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1440	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1441	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1442	}
1443	}
1444	else
1445	{
1446	GCPdPt = CPUMGetGuestCR3(pVCpu);
1447	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1448	}
1449
1450	/* Create a reference back to the PDPT by using the index in its shadow page. */
1451	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1452	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1453	&pShwPage);
1454	AssertRCReturn(rc, rc);
1455
1456	/* Hook it up. */
1457	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1458	}
1459	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1460
1461	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1462	return VINF_SUCCESS;
1463	}
1464
1465
1466	/**
1467	* Gets the pointer to the shadow page directory entry for an address, PAE.
1468	*
1469	* @returns Pointer to the PDE.
1470	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1471	* @param GCPtr The address.
1472	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1473	*/
1474	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1475	{
1476	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1477	PGM_LOCK_ASSERT_OWNER(pVM);
1478
1479	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1480	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1481	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1482	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1483	if (!(uPdpe & X86_PDPE_P))
1484	{
1485	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1486	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1487	}
1488	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1489
1490	/* Fetch the pgm pool shadow descriptor. */
1491	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1492	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1493
1494	*ppShwPde = pShwPde;
1495	return VINF_SUCCESS;
1496	}
1497
1498
1499	/**
1500	* Syncs the SHADOW page directory pointer for the specified address.
1501	*
1502	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1503	*
1504	* The caller is responsible for making sure the guest has a valid PD before
1505	* calling this function.
1506	*
1507	* @returns VBox status code.
1508	* @param pVCpu The cross context virtual CPU structure.
1509	* @param GCPtr The address.
1510	* @param uGstPml4e Guest PML4 entry (valid).
1511	* @param uGstPdpe Guest PDPT entry (valid).
1512	* @param ppPD Receives address of page directory
1513	*/
1514	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1515	{
1516	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1517	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1518	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1519	int rc;
1520
1521	PGM_LOCK_ASSERT_OWNER(pVM);
1522
1523	/*
1524	* PML4.
1525	*/
1526	PPGMPOOLPAGE pShwPage;
1527	{
1528	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1529	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1530	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1531	X86PGPAEUINT const uPml4e = pPml4e->u;
1532
1533	/* Allocate page directory pointer table if not present. */
1534	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1535	{
1536	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1537	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1538
1539	pgmPoolCacheUsed(pPool, pShwPage);
1540
1541	/* Update the entry if needed. */
1542	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1543	\| (uPml4e & PGM_PML4_FLAGS);
1544	if (uPml4e == uPml4eNew)
1545	{ /* likely */ }
1546	else
1547	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1548	}
1549	else
1550	{
1551	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1552
1553	RTGCPTR64 GCPml4;
1554	PGMPOOLKIND enmKind;
1555	if (fNestedPagingOrNoGstPaging)
1556	{
1557	/* AMD-V nested paging or real/protected mode without paging */
1558	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
1559	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1560	}
1561	else
1562	{
1563	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1564	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1565	}
1566
1567	/* Create a reference back to the PDPT by using the index in its shadow page. */
1568	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1569	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1570	&pShwPage);
1571	AssertRCReturn(rc, rc);
1572
1573	/* Hook it up. */
1574	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1575	\| (uPml4e & PGM_PML4_FLAGS));
1576	}
1577	}
1578
1579	/*
1580	* PDPT.
1581	*/
1582	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1583	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1584	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1585	X86PGPAEUINT const uPdpe = pPdpe->u;
1586
1587	/* Allocate page directory if not present. */
1588	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1589	{
1590	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1591	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1592
1593	pgmPoolCacheUsed(pPool, pShwPage);
1594
1595	/* Update the entry if needed. */
1596	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1597	\| (uPdpe & PGM_PDPT_FLAGS);
1598	if (uPdpe == uPdpeNew)
1599	{ /* likely */ }
1600	else
1601	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1602	}
1603	else
1604	{
1605	RTGCPTR64 GCPdPt;
1606	PGMPOOLKIND enmKind;
1607	if (fNestedPagingOrNoGstPaging)
1608	{
1609	/* AMD-V nested paging or real/protected mode without paging */
1610	GCPdPt = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
1611	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1612	}
1613	else
1614	{
1615	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1616	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1617	}
1618
1619	/* Create a reference back to the PDPT by using the index in its shadow page. */
1620	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1621	pShwPage->idx, iPdPt, false /fLockPage/,
1622	&pShwPage);
1623	AssertRCReturn(rc, rc);
1624
1625	/* Hook it up. */
1626	ASMAtomicWriteU64(&pPdpe->u,
1627	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1628	}
1629
1630	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1631	return VINF_SUCCESS;
1632	}
1633
1634
1635	/**
1636	* Gets the SHADOW page directory pointer for the specified address (long mode).
1637	*
1638	* @returns VBox status code.
1639	* @param pVCpu The cross context virtual CPU structure.
1640	* @param GCPtr The address.
1641	* @param ppPml4e Receives the address of the page map level 4 entry.
1642	* @param ppPdpt Receives the address of the page directory pointer table.
1643	* @param ppPD Receives the address of the page directory.
1644	*/
1645	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1646	{
1647	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1648	PGM_LOCK_ASSERT_OWNER(pVM);
1649
1650	/*
1651	* PML4
1652	*/
1653	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1654	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1655	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1656	if (ppPml4e)
1657	*ppPml4e = (PX86PML4E)pPml4e;
1658	X86PGPAEUINT const uPml4e = pPml4e->u;
1659	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1660	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1661	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1662
1663	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1664	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1665	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1666
1667	/*
1668	* PDPT
1669	*/
1670	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1671	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1672	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1673	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1674	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1675
1676	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1677	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1678
1679	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1680	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1681	return VINF_SUCCESS;
1682	}
1683
1684
1685	/**
1686	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1687	* backing pages in case the PDPT or PML4 entry is missing.
1688	*
1689	* @returns VBox status code.
1690	* @param pVCpu The cross context virtual CPU structure.
1691	* @param GCPtr The address.
1692	* @param ppPdpt Receives address of pdpt
1693	* @param ppPD Receives address of page directory
1694	*/
1695	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1696	{
1697	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1698	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1699	int rc;
1700
1701	Assert(pVM->pgm.s.fNestedPaging);
1702	PGM_LOCK_ASSERT_OWNER(pVM);
1703
1704	/*
1705	* PML4 level.
1706	*/
1707	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1708	Assert(pPml4);
1709
1710	/* Allocate page directory pointer table if not present. */
1711	PPGMPOOLPAGE pShwPage;
1712	{
1713	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1714	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1715	EPTPML4E Pml4e;
1716	Pml4e.u = pPml4e->u;
1717	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1718	{
1719	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1720	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1721	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1722	&pShwPage);
1723	AssertRCReturn(rc, rc);
1724
1725	/* Hook up the new PDPT now. */
1726	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1727	}
1728	else
1729	{
1730	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1731	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1732
1733	pgmPoolCacheUsed(pPool, pShwPage);
1734
1735	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1736	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1737	{ }
1738	else
1739	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1740	}
1741	}
1742
1743	/*
1744	* PDPT level.
1745	*/
1746	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1747	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1748	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1749
1750	if (ppPdpt)
1751	*ppPdpt = pPdpt;
1752
1753	/* Allocate page directory if not present. */
1754	EPTPDPTE Pdpe;
1755	Pdpe.u = pPdpe->u;
1756	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1757	{
1758	RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
1759	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1760	pShwPage->idx, iPdPt, false /fLockPage/,
1761	&pShwPage);
1762	AssertRCReturn(rc, rc);
1763
1764	/* Hook up the new PD now. */
1765	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1766	}
1767	else
1768	{
1769	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1770	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1771
1772	pgmPoolCacheUsed(pPool, pShwPage);
1773
1774	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1775	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1776	{ }
1777	else
1778	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1779	}
1780
1781	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1782	return VINF_SUCCESS;
1783	}
1784
1785
1786	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
1787	/**
1788	* Syncs the SHADOW nested-guest page directory pointer for the specified address.
1789	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1790	*
1791	* @returns VBox status code.
1792	* @param pVCpu The cross context virtual CPU structure.
1793	* @param GCPhysNested The nested-guest physical address.
1794	* @param ppPdpt Where to store the PDPT. Optional, can be NULL.
1795	* @param ppPD Where to store the PD. Optional, can be NULL.
1796	* @param pGstWalkAll The guest walk info.
1797	*/
1798	static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT ppPdpt, PEPTPD ppPD,
1799	PPGMPTWALKGST pGstWalkAll)
1800	{
1801	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1802	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1803	int rc;
1804
1805	PPGMPOOLPAGE pShwPage;
1806	Assert(pVM->pgm.s.fNestedPaging);
1807	Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
1808	PGM_LOCK_ASSERT_OWNER(pVM);
1809
1810	/*
1811	* PML4 level.
1812	*/
1813	{
1814	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1815	Assert(pPml4);
1816
1817	/* Allocate page directory pointer table if not present. */
1818	{
1819	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
1820	const unsigned iPml4e = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1821	PEPTPML4E pPml4e = &pPml4->a[iPml4e];
1822
1823	if (!(pPml4e->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1824	{
1825	RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
1826	rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
1827	PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /fLockPage/,
1828	&pShwPage);
1829	AssertRCReturn(rc, rc);
1830
1831	/* Hook up the new PDPT now. */
1832	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1833	}
1834	else
1835	{
1836	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1837	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1838
1839	pgmPoolCacheUsed(pPool, pShwPage);
1840
1841	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1842	if (pPml4e->u != (pShwPage->Core.Key \| fShwFlags))
1843	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| fShwFlags);
1844	}
1845	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1846	Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
1847	}
1848	}
1849
1850	/*
1851	* PDPT level.
1852	*/
1853	{
1854	AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
1855
1856	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1857	if (ppPdpt)
1858	*ppPdpt = pPdpt;
1859
1860	uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
1861	const unsigned iPdPte = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1862	PEPTPDPTE pPdpte = &pPdpt->a[iPdPte];
1863
1864	if (!(pPdpte->u & (EPT_E_PG_MASK \| EPT_PRESENT_MASK)))
1865	{
1866	RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
1867	rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1868	pShwPage->idx, iPdPte, false /fLockPage/, &pShwPage);
1869	AssertRCReturn(rc, rc);
1870
1871	/* Hook up the new PD now. */
1872	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1873	}
1874	else
1875	{
1876	pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
1877	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1878
1879	pgmPoolCacheUsed(pPool, pShwPage);
1880
1881	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1882	if (pPdpte->u != (pShwPage->Core.Key \| fShwFlags))
1883	ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key \| fShwFlags);
1884	}
1885	Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
1886	Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
1887
1888	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1889	}
1890
1891	return VINF_SUCCESS;
1892	}
1893	# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
1894
1895
1896	# ifdef IN_RING0
1897	/**
1898	* Synchronizes a range of nested page table entries.
1899	*
1900	* The caller must own the PGM lock.
1901	*
1902	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1903	* @param GCPhys Where to start.
1904	* @param cPages How many pages which entries should be synced.
1905	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1906	* host paging mode for AMD-V).
1907	*/
1908	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1909	{
1910	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1911
1912	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1913	int rc;
1914	switch (enmShwPagingMode)
1915	{
1916	case PGMMODE_32_BIT:
1917	{
1918	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1919	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1920	break;
1921	}
1922
1923	case PGMMODE_PAE:
1924	case PGMMODE_PAE_NX:
1925	{
1926	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1927	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1928	break;
1929	}
1930
1931	case PGMMODE_AMD64:
1932	case PGMMODE_AMD64_NX:
1933	{
1934	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1935	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1936	break;
1937	}
1938
1939	case PGMMODE_EPT:
1940	{
1941	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1942	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1943	break;
1944	}
1945
1946	default:
1947	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1948	}
1949	return rc;
1950	}
1951	# endif /* IN_RING0 */
1952
1953	# endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
1954
1955	#endif /* VBOX_VMM_TARGET_X86 */
1956
1957
1958	/**
1959	* Gets effective Guest OS page information.
1960	*
1961	* @returns VBox status code.
1962	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1963	* @param GCPtr Guest Context virtual address of the page.
1964	* @param pWalk Where to store the page walk information.
1965	* @thread EMT(pVCpu)
1966	*/
1967	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
1968	{
1969	VMCPU_ASSERT_EMT(pVCpu);
1970	Assert(pWalk);
1971	#ifdef VBOX_VMM_TARGET_X86
1972	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1973	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1974	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1975	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
1976
1977	#elif defined(VBOX_VMM_TARGET_ARMV8)
1978	return pgmGstGetPageArmv8Hack(pVCpu, GCPtr, pWalk);
1979	#else
1980	# error "port me"
1981	#endif
1982	}
1983
1984	#ifdef VBOX_VMM_TARGET_X86
1985
1986	/**
1987	* Gets effective Guest OS page information.
1988	*
1989	* @returns VBox status code.
1990	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1991	* @param GCPtr Guest Context virtual address of the page.
1992	* @param fFlags PGMQPAGE_F_XXX. If zero, no accessed or dirty bits will
1993	* be set.
1994	* @param pWalk Where to store the page walk information.
1995	* @thread EMT(pVCpu)
1996	*/
1997	VMM_INT_DECL(int) PGMGstQueryPageFast(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fFlags, PPGMPTWALKFAST pWalk)
1998	{
1999	VMCPU_ASSERT_EMT(pVCpu);
2000	Assert(pWalk);
2001	Assert(!(fFlags & ~(PGMQPAGE_F_VALID_MASK)));
2002	Assert(!(fFlags & PGMQPAGE_F_EXECUTE) \|\| !(fFlags & PGMQPAGE_F_WRITE));
2003	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2004	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2005	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
2006	return g_aPgmGuestModeData[idx].pfnQueryPageFast(pVCpu, GCPtr, fFlags, pWalk);
2007	}
2008
2009
2010	/**
2011	* Maps the guest CR3.
2012	*
2013	* @returns VBox status code.
2014	* @param pVCpu The cross context virtual CPU structure.
2015	* @param GCPhysCr3 The guest CR3 value.
2016	* @param pHCPtrGuestCr3 Where to store the mapped memory.
2017	*/
2018	DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
2019	{
2020	/** @todo this needs some reworking wrt. locking? */
2021	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2022	PGM_LOCK_VOID(pVM);
2023	PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
2024	AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
2025
2026	RTHCPTR HCPtrGuestCr3;
2027	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
2028	PGM_UNLOCK(pVM);
2029
2030	*pHCPtrGuestCr3 = HCPtrGuestCr3;
2031	return rc;
2032	}
2033
2034
2035	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2036	/**
2037	* Unmaps the guest CR3.
2038	*
2039	* @returns VBox status code.
2040	* @param pVCpu The cross context virtual CPU structure.
2041	*/
2042	DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
2043	{
2044	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2045	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2046	AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
2047	return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
2048	}
2049	# endif
2050
2051	#endif /* VBOX_VMM_TARGET_X86 */
2052
2053
2054	/**
2055	* Performs a guest page table walk.
2056	*
2057	* The guest should be in paged protect mode or long mode when making a call to
2058	* this function.
2059	*
2060	* @returns VBox status code.
2061	* @retval VINF_SUCCESS on success.
2062	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2063	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2064	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2065	*
2066	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2067	* @param GCPtr The guest virtual address to walk by.
2068	* @param pWalk Where to return the walk result. This is valid for some
2069	* error codes as well.
2070	* @param pGstWalk The guest mode specific page walk information.
2071	*/
2072	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2073	{
2074	VMCPU_ASSERT_EMT(pVCpu);
2075	#ifdef VBOX_VMM_TARGET_X86
2076	switch (pVCpu->pgm.s.enmGuestMode)
2077	{
2078	case PGMMODE_32_BIT:
2079	pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
2080	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
2081
2082	case PGMMODE_PAE:
2083	case PGMMODE_PAE_NX:
2084	pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
2085	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
2086
2087	case PGMMODE_AMD64:
2088	case PGMMODE_AMD64_NX:
2089	pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
2090	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
2091
2092	case PGMMODE_REAL:
2093	case PGMMODE_PROTECTED:
2094	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2095	return VERR_PGM_NOT_USED_IN_MODE;
2096
2097	case PGMMODE_EPT:
2098	case PGMMODE_NESTED_32BIT:
2099	case PGMMODE_NESTED_PAE:
2100	case PGMMODE_NESTED_AMD64:
2101	default:
2102	AssertFailed();
2103	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2104	return VERR_PGM_NOT_USED_IN_MODE;
2105	}
2106
2107	#elif defined(VBOX_VMM_TARGET_ARMV8)
2108	/** @todo temporary hack. */
2109	RT_NOREF(pGstWalk);
2110	return pgmGstGetPageArmv8Hack(pVCpu, GCPtr, pWalk);
2111
2112	#else
2113	# error "port me"
2114	#endif
2115	}
2116
2117
2118	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2119	/**
2120	* Performs a guest second-level address translation (SLAT).
2121	*
2122	* @returns VBox status code.
2123	* @retval VINF_SUCCESS on success.
2124	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2125	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2126	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2127	*
2128	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2129	* @param GCPhysNested The nested-guest physical address being translated.
2130	* @param fIsLinearAddrValid Whether the linear address in @a GCPtrNested is the
2131	* cause for this translation.
2132	* @param GCPtrNested The nested-guest virtual address that initiated the
2133	* SLAT. If none, pass 0 (and not NIL_RTGCPTR).
2134	* @param pWalk Where to return the walk result. This is updated for
2135	* all error codes other than
2136	* VERR_PGM_NOT_USED_IN_MODE.
2137	* @param pGstWalk Where to store the second-level paging-mode specific
2138	* walk info.
2139	*/
2140	static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
2141	PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2142	{
2143	/* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
2144	Assert( pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
2145	&& pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
2146	AssertPtr(pWalk);
2147	AssertPtr(pGstWalk);
2148	switch (pVCpu->pgm.s.enmGuestSlatMode)
2149	{
2150	case PGMSLAT_EPT:
2151	pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
2152	return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
2153
2154	default:
2155	AssertFailed();
2156	pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
2157	return VERR_PGM_NOT_USED_IN_MODE;
2158	}
2159	}
2160	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
2161
2162
2163	/**
2164	* Tries to continue the previous walk.
2165	*
2166	* @note Requires the caller to hold the PGM lock from the first
2167	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
2168	* we cannot use the pointers.
2169	*
2170	* @returns VBox status code.
2171	* @retval VINF_SUCCESS on success.
2172	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
2173	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
2174	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
2175	*
2176	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2177	* @param GCPtr The guest virtual address to walk by.
2178	* @param pWalk Pointer to the previous walk result and where to return
2179	* the result of this walk. This is valid for some error
2180	* codes as well.
2181	* @param pGstWalk The guest-mode specific walk information.
2182	*/
2183	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
2184	{
2185	#ifdef VBOX_VMM_TARGET_X86 /** @todo optimize for ARMv8 */
2186	/*
2187	* We can only handle successfully walks.
2188	* We also limit ourselves to the next page.
2189	*/
2190	if ( pWalk->fSucceeded
2191	&& GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
2192	{
2193	Assert(pWalk->uLevel == 0);
2194	if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2195	{
2196	/*
2197	* AMD64
2198	*/
2199	if (!pWalk->fGigantPage && !pWalk->fBigPage)
2200	{
2201	/*
2202	* We fall back to full walk if the PDE table changes, if any
2203	* reserved bits are set, or if the effective page access changes.
2204	*/
2205	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
2206	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
2207	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
2208	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
2209
2210	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
2211	{
2212	if (pGstWalk->u.Amd64.pPte)
2213	{
2214	X86PTEPAE Pte;
2215	Pte.u = pGstWalk->u.Amd64.pPte[1].u;
2216	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2217	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2218	{
2219	pWalk->GCPtr = GCPtr;
2220	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2221	pGstWalk->u.Amd64.Pte.u = Pte.u;
2222	pGstWalk->u.Amd64.pPte++;
2223	return VINF_SUCCESS;
2224	}
2225	}
2226	}
2227	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
2228	{
2229	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2230	if (pGstWalk->u.Amd64.pPde)
2231	{
2232	X86PDEPAE Pde;
2233	Pde.u = pGstWalk->u.Amd64.pPde[1].u;
2234	if ( (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
2235	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2236	{
2237	/* Get the new PTE and check out the first entry. */
2238	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2239	&pGstWalk->u.Amd64.pPt);
2240	if (RT_SUCCESS(rc))
2241	{
2242	pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
2243	X86PTEPAE Pte;
2244	Pte.u = pGstWalk->u.Amd64.pPte->u;
2245	if ( (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
2246	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2247	{
2248	pWalk->GCPtr = GCPtr;
2249	pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2250	pGstWalk->u.Amd64.Pte.u = Pte.u;
2251	pGstWalk->u.Amd64.Pde.u = Pde.u;
2252	pGstWalk->u.Amd64.pPde++;
2253	return VINF_SUCCESS;
2254	}
2255	}
2256	}
2257	}
2258	}
2259	}
2260	else if (!pWalk->fGigantPage)
2261	{
2262	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
2263	{
2264	pWalk->GCPtr = GCPtr;
2265	pWalk->GCPhys += GUEST_PAGE_SIZE;
2266	return VINF_SUCCESS;
2267	}
2268	}
2269	else
2270	{
2271	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
2272	{
2273	pWalk->GCPtr = GCPtr;
2274	pWalk->GCPhys += GUEST_PAGE_SIZE;
2275	return VINF_SUCCESS;
2276	}
2277	}
2278	}
2279	}
2280	#endif /* VBOX_VMM_TARGET_X86 */
2281	/* Case we don't handle. Do full walk. */
2282	return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
2283	}
2284
2285
2286	#ifdef VBOX_VMM_TARGET_X86
2287	/**
2288	* Modify page flags for a range of pages in the guest's tables
2289	*
2290	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2291	*
2292	* @returns VBox status code.
2293	* @param pVCpu The cross context virtual CPU structure.
2294	* @param GCPtr Virtual address of the first page in the range.
2295	* @param cb Size (in bytes) of the range to apply the modification to.
2296	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2297	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2298	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2299	*/
2300	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2301	{
2302	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2303	VMCPU_ASSERT_EMT(pVCpu);
2304
2305	/*
2306	* Validate input.
2307	*/
2308	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2309	Assert(cb);
2310
2311	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2312
2313	/*
2314	* Adjust input.
2315	*/
2316	cb += GCPtr & GUEST_PAGE_OFFSET_MASK;
2317	cb = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
2318	GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
2319
2320	/*
2321	* Call worker.
2322	*/
2323	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2324	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2325	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2326	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2327
2328	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2329	return rc;
2330	}
2331	#endif /* VBOX_VMM_TARGET_X86 */
2332
2333	#ifdef VBOX_VMM_TARGET_X86
2334
2335	/**
2336	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2337	*
2338	* @returns @c true if the PDPE is valid, @c false otherwise.
2339	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2340	* @param paPaePdpes The PAE PDPEs to validate.
2341	*
2342	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2343	*/
2344	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2345	{
2346	Assert(paPaePdpes);
2347	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2348	{
2349	X86PDPE const PaePdpe = paPaePdpes[i];
2350	if ( !(PaePdpe.u & X86_PDPE_P)
2351	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2352	{ /* likely */ }
2353	else
2354	return false;
2355	}
2356	return true;
2357	}
2358
2359
2360	/**
2361	* Performs the lazy mapping of the 32-bit guest PD.
2362	*
2363	* @returns VBox status code.
2364	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2365	* @param ppPd Where to return the pointer to the mapping. This is
2366	* always set.
2367	*/
2368	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2369	{
2370	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2371	PGM_LOCK_VOID(pVM);
2372
2373	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2374
2375	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2376	PPGMPAGE pPage;
2377	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2378	if (RT_SUCCESS(rc))
2379	{
2380	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2381	if (RT_SUCCESS(rc))
2382	{
2383	# ifdef IN_RING3
2384	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2385	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2386	# else
2387	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2388	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2389	# endif
2390	PGM_UNLOCK(pVM);
2391	return VINF_SUCCESS;
2392	}
2393	AssertRC(rc);
2394	}
2395	PGM_UNLOCK(pVM);
2396
2397	*ppPd = NULL;
2398	return rc;
2399	}
2400
2401
2402	/**
2403	* Performs the lazy mapping of the PAE guest PDPT.
2404	*
2405	* @returns VBox status code.
2406	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2407	* @param ppPdpt Where to return the pointer to the mapping. This is
2408	* always set.
2409	*/
2410	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2411	{
2412	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2413	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2414	PGM_LOCK_VOID(pVM);
2415
2416	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2417	PPGMPAGE pPage;
2418	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2419	if (RT_SUCCESS(rc))
2420	{
2421	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2422	if (RT_SUCCESS(rc))
2423	{
2424	# ifdef IN_RING3
2425	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2426	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2427	# else
2428	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2429	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2430	# endif
2431	PGM_UNLOCK(pVM);
2432	return VINF_SUCCESS;
2433	}
2434	AssertRC(rc);
2435	}
2436
2437	PGM_UNLOCK(pVM);
2438	*ppPdpt = NULL;
2439	return rc;
2440	}
2441
2442
2443	/**
2444	* Performs the lazy mapping / updating of a PAE guest PD.
2445	*
2446	* @returns Pointer to the mapping.
2447	* @returns VBox status code.
2448	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2449	* @param iPdpt Which PD entry to map (0..3).
2450	* @param ppPd Where to return the pointer to the mapping. This is
2451	* always set.
2452	*/
2453	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2454	{
2455	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2456	PGM_LOCK_VOID(pVM);
2457
2458	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2459	Assert(pGuestPDPT);
2460	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2461	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2462	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2463
2464	PPGMPAGE pPage;
2465	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2466	if (RT_SUCCESS(rc))
2467	{
2468	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2469	AssertRC(rc);
2470	if (RT_SUCCESS(rc))
2471	{
2472	# ifdef IN_RING3
2473	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2474	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2475	# else
2476	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2477	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2478	# endif
2479	if (fChanged)
2480	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2481	PGM_UNLOCK(pVM);
2482	return VINF_SUCCESS;
2483	}
2484	}
2485
2486	/* Invalid page or some failure, invalidate the entry. */
2487	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2488	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2489	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2490
2491	PGM_UNLOCK(pVM);
2492	return rc;
2493	}
2494
2495
2496	/**
2497	* Performs the lazy mapping of the 32-bit guest PD.
2498	*
2499	* @returns VBox status code.
2500	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2501	* @param ppPml4 Where to return the pointer to the mapping. This will
2502	* always be set.
2503	*/
2504	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2505	{
2506	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2507	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2508	PGM_LOCK_VOID(pVM);
2509
2510	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
2511	PPGMPAGE pPage;
2512	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2513	if (RT_SUCCESS(rc))
2514	{
2515	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2516	if (RT_SUCCESS(rc))
2517	{
2518	# ifdef IN_RING3
2519	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2520	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2521	# else
2522	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2523	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2524	# endif
2525	PGM_UNLOCK(pVM);
2526	return VINF_SUCCESS;
2527	}
2528	}
2529
2530	PGM_UNLOCK(pVM);
2531	*ppPml4 = NULL;
2532	return rc;
2533	}
2534
2535
2536	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2537	/**
2538	* Performs the lazy mapping of the guest PML4 table when using EPT paging.
2539	*
2540	* @returns VBox status code.
2541	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2542	* @param ppEptPml4 Where to return the pointer to the mapping. This will
2543	* always be set.
2544	*/
2545	int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
2546	{
2547	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
2548	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2549	PGM_LOCK_VOID(pVM);
2550
2551	RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
2552	PPGMPAGE pPage;
2553	int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
2554	if (RT_SUCCESS(rc))
2555	{
2556	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
2557	if (RT_SUCCESS(rc))
2558	{
2559	# ifdef IN_RING3
2560	pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
2561	pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
2562	# else
2563	pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
2564	pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
2565	# endif
2566	PGM_UNLOCK(pVM);
2567	return VINF_SUCCESS;
2568	}
2569	}
2570
2571	PGM_UNLOCK(pVM);
2572	*ppEptPml4 = NULL;
2573	return rc;
2574	}
2575	# endif
2576
2577
2578	/**
2579	* Gets the current CR3 register value for the shadow memory context.
2580	* @returns CR3 value.
2581	* @param pVCpu The cross context virtual CPU structure.
2582	*/
2583	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2584	{
2585	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2586	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2587	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2588	return pPoolPage->Core.Key;
2589	# else
2590	RT_NOREF(pVCpu);
2591	return NIL_RTHCPHYS;
2592	# endif
2593	}
2594
2595
2596	/**
2597	* Forces lazy remapping of the guest's PAE page-directory structures.
2598	*
2599	* @param pVCpu The cross context virtual CPU structure.
2600	*/
2601	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2602	{
2603	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2604	{
2605	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2606	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2607	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2608	}
2609	}
2610
2611
2612	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2613	/**
2614	* Performs second-level address translation for the given CR3 and updates the
2615	* nested-guest CR3 when successful.
2616	*
2617	* @returns VBox status code.
2618	* @param pVCpu The cross context virtual CPU structure.
2619	* @param uCr3 The masked nested-guest CR3 value.
2620	* @param pGCPhysCR3 Where to store the translated CR3.
2621	*
2622	* @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
2623	* mindful of this in code that's hyper sensitive to the order of
2624	* operations.
2625	*/
2626	static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
2627	{
2628	if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
2629	{
2630	PGMPTWALK Walk;
2631	PGMPTWALKGST GstWalk;
2632	int const rc = pgmGstSlatWalk(pVCpu, uCr3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk, &GstWalk);
2633	if (RT_SUCCESS(rc))
2634	{
2635	/* Update nested-guest CR3. */
2636	pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
2637
2638	/* Pass back the translated result. */
2639	*pGCPhysCr3 = Walk.GCPhys;
2640	return VINF_SUCCESS;
2641	}
2642
2643	/* Translation failed. */
2644	*pGCPhysCr3 = NIL_RTGCPHYS;
2645	return rc;
2646	}
2647
2648	/*
2649	* If the nested-guest CR3 has not changed, then the previously
2650	* translated CR3 result (i.e. GCPhysCR3) is passed back.
2651	*/
2652	*pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
2653	return VINF_SUCCESS;
2654	}
2655	# endif
2656
2657
2658	/**
2659	* Performs and schedules necessary updates following a CR3 load or reload.
2660	*
2661	* This will normally involve mapping the guest PD or nPDPT
2662	*
2663	* @returns VBox status code.
2664	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2665	* safely be ignored and overridden since the FF will be set too then.
2666	* @param pVCpu The cross context virtual CPU structure.
2667	* @param cr3 The new cr3.
2668	* @param fGlobal Indicates whether this is a global flush or not.
2669	*/
2670	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2671	{
2672	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2673
2674	VMCPU_ASSERT_EMT(pVCpu);
2675
2676	/*
2677	* Always flag the necessary updates; necessary for hardware acceleration
2678	*/
2679	/** @todo optimize this, it shouldn't always be necessary. */
2680	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2681	if (fGlobal)
2682	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2683
2684	/*
2685	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2686	*/
2687	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2688	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2689	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2690	if ( pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
2691	&& PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
2692	{
2693	RTGCPHYS GCPhysOut;
2694	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2695	if (RT_SUCCESS(rc))
2696	GCPhysCR3 = GCPhysOut;
2697	else
2698	{
2699	/* CR3 SLAT translation failed but we try to pretend it
2700	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2701	AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2702	int const rc2 = pgmGstUnmapCr3(pVCpu);
2703	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2704	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2705	return rc2;
2706	}
2707	}
2708	#endif
2709
2710	LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
2711	int rc = VINF_SUCCESS;
2712	if (GCPhysOldCR3 != GCPhysCR3)
2713	{
2714	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2715	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2716	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2717
2718	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2719	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2720	if (RT_LIKELY(rc == VINF_SUCCESS))
2721	{ }
2722	else
2723	{
2724	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2725	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2726	pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
2727	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
2728	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2729	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2730	}
2731
2732	if (fGlobal)
2733	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2734	else
2735	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2736	}
2737	else
2738	{
2739	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2740	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
2741	PPGMPOOL const pPool = pVM->pgm.s.CTX_SUFF(pPool);
2742	if (pPool->cDirtyPages)
2743	{
2744	PGM_LOCK_VOID(pVM);
2745	pgmPoolResetDirtyPages(pVM);
2746	PGM_UNLOCK(pVM);
2747	}
2748	#endif
2749	if (fGlobal)
2750	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2751	else
2752	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2753
2754	/*
2755	* Flush PAE PDPTEs.
2756	*/
2757	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2758	pgmGstFlushPaePdpes(pVCpu);
2759	}
2760
2761	if (!fGlobal)
2762	IEMTlbInvalidateAll(pVCpu);
2763	else
2764	IEMTlbInvalidateAllGlobal(pVCpu);
2765	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2766	return rc;
2767	}
2768
2769
2770	/**
2771	* Performs and schedules necessary updates following a CR3 load or reload when
2772	* using nested or extended paging.
2773	*
2774	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2775	* TLB and triggering a SyncCR3.
2776	*
2777	* This will normally involve mapping the guest PD or nPDPT
2778	*
2779	* @returns VBox status code.
2780	* @retval VINF_SUCCESS.
2781	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2782	* paging modes). This can safely be ignored and overridden since the
2783	* FF will be set too then.
2784	* @param pVCpu The cross context virtual CPU structure.
2785	* @param cr3 The new CR3.
2786	*/
2787	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2788	{
2789	VMCPU_ASSERT_EMT(pVCpu);
2790
2791	/* We assume we're only called in nested paging mode. */
2792	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2793
2794	/*
2795	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2796	*/
2797	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2798	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2799	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2800	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2801	{
2802	RTGCPHYS GCPhysOut;
2803	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2804	if (RT_SUCCESS(rc))
2805	GCPhysCR3 = GCPhysOut;
2806	else
2807	{
2808	/* CR3 SLAT translation failed but we try to pretend it
2809	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2810	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
2811	int const rc2 = pgmGstUnmapCr3(pVCpu);
2812	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2813	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2814	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2815	return rc2;
2816	}
2817	}
2818	#endif
2819
2820	LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
2821	int rc = VINF_SUCCESS;
2822	if (GCPhysOldCR3 != GCPhysCR3)
2823	{
2824	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2825	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2826	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2827
2828	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2829	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2830
2831	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2832	}
2833	/*
2834	* Flush PAE PDPTEs.
2835	*/
2836	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2837	pgmGstFlushPaePdpes(pVCpu);
2838
2839	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2840	return rc;
2841	}
2842
2843
2844	/**
2845	* Synchronize the paging structures.
2846	*
2847	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2848	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2849	* in several places, most importantly whenever the CR3 is loaded.
2850	*
2851	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2852	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2853	* the VMM into guest context.
2854	* @param pVCpu The cross context virtual CPU structure.
2855	* @param cr0 Guest context CR0 register
2856	* @param cr3 Guest context CR3 register
2857	* @param cr4 Guest context CR4 register
2858	* @param fGlobal Including global page directories or not
2859	*/
2860	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2861	{
2862	VMCPU_ASSERT_EMT(pVCpu);
2863
2864	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2865	/*
2866	* The pool may have pending stuff and even require a return to ring-3 to
2867	* clear the whole thing.
2868	*/
2869	int rcPool = pgmPoolSyncCR3(pVCpu);
2870	if (rcPool != VINF_SUCCESS)
2871	return rcPool;
2872	#endif
2873
2874	/*
2875	* We might be called when we shouldn't.
2876	*
2877	* The mode switching will ensure that the PD is resynced after every mode
2878	* switch. So, if we find ourselves here when in protected or real mode
2879	* we can safely clear the FF and return immediately.
2880	*/
2881	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2882	{
2883	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2884	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2886	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2887	return VINF_SUCCESS;
2888	}
2889
2890	/* If global pages are not supported, then all flushes are global. */
2891	if (!(cr4 & X86_CR4_PGE))
2892	fGlobal = true;
2893	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2894	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2895
2896	/*
2897	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2898	* This should be done before SyncCR3.
2899	*/
2900	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2901	{
2902	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2903
2904	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2905	RTGCPHYS GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2906	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
2907	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
2908	{
2909	RTGCPHYS GCPhysOut;
2910	int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
2911	if (RT_SUCCESS(rc2))
2912	GCPhysCR3 = GCPhysOut;
2913	else
2914	{
2915	/* CR3 SLAT translation failed but we try to pretend it
2916	succeeded for the reasons mentioned in PGMHCChangeMode(). */
2917	AssertMsgFailed(("Failed to translate CR3 %#RX64. rc2=%Rrc\n", cr3, rc2));
2918	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
2919	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
2920	return rc2;
2921	}
2922	}
2923	#endif
2924	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
2925	int rc = VINF_SUCCESS;
2926	if (GCPhysOldCR3 != GCPhysCR3)
2927	{
2928	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2929	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2930	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2931	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2932	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2933	}
2934
2935	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2936	if ( rc == VINF_PGM_SYNC_CR3
2937	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2938	{
2939	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2940	#ifdef IN_RING3
2941	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2942	rc = pgmPoolSyncCR3(pVCpu);
2943	# else
2944	rc = VINF_SUCCESS;
2945	# endif
2946	#else
2947	if (rc == VINF_PGM_SYNC_CR3)
2948	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2949	return VINF_PGM_SYNC_CR3;
2950	#endif
2951	}
2952	AssertRCReturn(rc, rc);
2953	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2954	}
2955
2956	/*
2957	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2958	*/
2959	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2960
2961	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2962	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2963	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2964	int rcSync = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2965
2966	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2967	AssertMsg(rcSync == VINF_SUCCESS \|\| rcSync == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rcSync), ("rcSync=%Rrc\n", rcSync));
2968	if (rcSync == VINF_SUCCESS)
2969	{
2970	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2971	{
2972	/* Go back to ring 3 if a pgm pool sync is again pending. */
2973	return VINF_PGM_SYNC_CR3;
2974	}
2975
2976	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2977	{
2978	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2979	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2980	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2981	}
2982	}
2983
2984	/*
2985	* Now flush the CR3 (guest context).
2986	*/
2987	if (rcSync == VINF_SUCCESS)
2988	PGM_INVL_VCPU_TLBS(pVCpu);
2989	return rcSync;
2990	}
2991
2992
2993	/**
2994	* Maps all the PAE PDPE entries.
2995	*
2996	* @returns VBox status code.
2997	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2998	* @param paPaePdpes The new PAE PDPE values.
2999	*
3000	* @remarks This function may be invoked during the process of changing the guest
3001	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
3002	* reflect PAE paging just yet.
3003	*/
3004	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
3005	{
3006	Assert(paPaePdpes);
3007	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
3008	{
3009	X86PDPE const PaePdpe = paPaePdpes[i];
3010
3011	/*
3012	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
3013	* are deferred.[1] Also, different situations require different handling of invalid
3014	* PDPE entries. Here we assume the caller has already validated or doesn't require
3015	* validation of the PDPEs.
3016	*
3017	* In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
3018	* validated by the VMX transition.
3019	*
3020	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
3021	*/
3022	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
3023	{
3024	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3025	RTHCPTR HCPtr;
3026
3027	RTGCPHYS GCPhys;
3028	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3029	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3030	{
3031	PGMPTWALK Walk;
3032	PGMPTWALKGST GstWalk;
3033	RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
3034	int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid /, 0 / GCPtrNested */,
3035	&Walk, &GstWalk);
3036	if (RT_SUCCESS(rc))
3037	GCPhys = Walk.GCPhys;
3038	else
3039	{
3040	/*
3041	* Second-level address translation of the PAE PDPE has failed but we must -NOT-
3042	* abort and return a failure now. This is because we're called from a Mov CRx
3043	* instruction (or similar operation). Let's just pretend success but flag that
3044	* we need to map this PDPE lazily later.
3045	*
3046	* See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
3047	* See Intel spec. 28.3.1 "EPT Overview".
3048	*/
3049	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3050	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3051	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3052	continue;
3053	}
3054	}
3055	else
3056	#endif
3057	{
3058	GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
3059	}
3060
3061	PGM_LOCK_VOID(pVM);
3062	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
3063	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
3064	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
3065	PGM_UNLOCK(pVM);
3066	if (RT_SUCCESS(rc))
3067	{
3068	#ifdef IN_RING3
3069	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
3070	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
3071	#else
3072	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
3073	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
3074	#endif
3075	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
3076	continue;
3077	}
3078	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
3079	}
3080	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
3081	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
3082	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
3083	}
3084	return VINF_SUCCESS;
3085	}
3086
3087
3088	/**
3089	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
3090	*
3091	* @returns VBox status code.
3092	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3093	* @param cr3 The guest CR3 value.
3094	*
3095	* @remarks This function may be invoked during the process of changing the guest
3096	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
3097	* PAE paging just yet.
3098	*/
3099	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
3100	{
3101	/*
3102	* Read the page-directory-pointer table (PDPT) at CR3.
3103	*/
3104	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
3105	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
3106
3107	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3108	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3109	{
3110	RTGCPHYS GCPhysOut;
3111	int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
3112	if (RT_SUCCESS(rc))
3113	GCPhysCR3 = GCPhysOut;
3114	else
3115	{
3116	Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
3117	return rc;
3118	}
3119	}
3120	#endif
3121
3122	RTHCPTR HCPtrGuestCr3;
3123	int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
3124	if (RT_SUCCESS(rc))
3125	{
3126	/*
3127	* Validate the page-directory-pointer table entries (PDPE).
3128	*/
3129	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
3130	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
3131	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
3132	{
3133	/*
3134	* Map the PDPT.
3135	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
3136	* that PGMFlushTLB will be called soon and only a change to CR3 then
3137	* will cause the shadow page tables to be updated.
3138	*/
3139	#ifdef IN_RING3
3140	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
3141	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
3142	#else
3143	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
3144	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
3145	#endif
3146
3147	/*
3148	* Update CPUM and map the 4 PAE PDPEs.
3149	*/
3150	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
3151	rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
3152	if (RT_SUCCESS(rc))
3153	{
3154	#ifdef IN_RING3
3155	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
3156	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
3157	#else
3158	pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
3159	pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
3160	#endif
3161	pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
3162	}
3163	}
3164	else
3165	rc = VERR_PGM_PAE_PDPE_RSVD;
3166	}
3167	return rc;
3168	}
3169
3170
3171	/**
3172	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
3173	*
3174	* @returns VBox status code, with the following informational code for
3175	* VM scheduling.
3176	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
3177	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
3178	*
3179	* @param pVCpu The cross context virtual CPU structure.
3180	* @param cr0 The new cr0.
3181	* @param cr4 The new cr4.
3182	* @param efer The new extended feature enable register.
3183	* @param fForce Whether to force a mode change.
3184	*/
3185	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
3186	{
3187	VMCPU_ASSERT_EMT(pVCpu);
3188
3189	/*
3190	* Calc the new guest mode.
3191	*
3192	* Note! We check PG before PE and without requiring PE because of the
3193	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
3194	*/
3195	PGMMODE enmGuestMode;
3196	if (cr0 & X86_CR0_PG)
3197	{
3198	if (!(cr4 & X86_CR4_PAE))
3199	{
3200	bool const fPse = !!(cr4 & X86_CR4_PSE);
3201	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
3202	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
3203	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
3204	enmGuestMode = PGMMODE_32_BIT;
3205	}
3206	else if (!(efer & MSR_K6_EFER_LME))
3207	{
3208	if (!(efer & MSR_K6_EFER_NXE))
3209	enmGuestMode = PGMMODE_PAE;
3210	else
3211	enmGuestMode = PGMMODE_PAE_NX;
3212	}
3213	else
3214	{
3215	if (!(efer & MSR_K6_EFER_NXE))
3216	enmGuestMode = PGMMODE_AMD64;
3217	else
3218	enmGuestMode = PGMMODE_AMD64_NX;
3219	}
3220	}
3221	else if (!(cr0 & X86_CR0_PE))
3222	enmGuestMode = PGMMODE_REAL;
3223	else
3224	enmGuestMode = PGMMODE_PROTECTED;
3225
3226	/*
3227	* Did it change?
3228	*/
3229	if ( !fForce
3230	&& pVCpu->pgm.s.enmGuestMode == enmGuestMode)
3231	return VINF_SUCCESS;
3232
3233	/* Flush the TLB */
3234	PGM_INVL_VCPU_TLBS(pVCpu);
3235	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
3236	}
3237
3238
3239	/**
3240	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
3241	*
3242	* @returns PGM_TYPE_*.
3243	* @param pgmMode The mode value to convert.
3244	*/
3245	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
3246	{
3247	switch (pgmMode)
3248	{
3249	case PGMMODE_REAL: return PGM_TYPE_REAL;
3250	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
3251	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
3252	case PGMMODE_PAE:
3253	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
3254	case PGMMODE_AMD64:
3255	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
3256	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
3257	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
3258	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
3259	case PGMMODE_EPT: return PGM_TYPE_EPT;
3260	case PGMMODE_NONE: return PGM_TYPE_NONE;
3261	default:
3262	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
3263	}
3264	}
3265
3266
3267	/**
3268	* Calculates the shadow paging mode.
3269	*
3270	* @returns The shadow paging mode.
3271	* @param pVM The cross context VM structure.
3272	* @param enmGuestMode The guest mode.
3273	* @param enmHostMode The host mode.
3274	* @param enmShadowMode The current shadow mode.
3275	*/
3276	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
3277	{
3278	switch (enmGuestMode)
3279	{
3280	case PGMMODE_REAL:
3281	case PGMMODE_PROTECTED:
3282	switch (enmHostMode)
3283	{
3284	case SUPPAGINGMODE_32_BIT:
3285	case SUPPAGINGMODE_32_BIT_GLOBAL:
3286	enmShadowMode = PGMMODE_32_BIT;
3287	break;
3288
3289	case SUPPAGINGMODE_PAE:
3290	case SUPPAGINGMODE_PAE_NX:
3291	case SUPPAGINGMODE_PAE_GLOBAL:
3292	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3293	enmShadowMode = PGMMODE_PAE;
3294	break;
3295
3296	case SUPPAGINGMODE_AMD64:
3297	case SUPPAGINGMODE_AMD64_GLOBAL:
3298	case SUPPAGINGMODE_AMD64_NX:
3299	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3300	enmShadowMode = PGMMODE_PAE;
3301	break;
3302
3303	default:
3304	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3305	}
3306	break;
3307
3308	case PGMMODE_32_BIT:
3309	switch (enmHostMode)
3310	{
3311	case SUPPAGINGMODE_32_BIT:
3312	case SUPPAGINGMODE_32_BIT_GLOBAL:
3313	enmShadowMode = PGMMODE_32_BIT;
3314	break;
3315
3316	case SUPPAGINGMODE_PAE:
3317	case SUPPAGINGMODE_PAE_NX:
3318	case SUPPAGINGMODE_PAE_GLOBAL:
3319	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3320	enmShadowMode = PGMMODE_PAE;
3321	break;
3322
3323	case SUPPAGINGMODE_AMD64:
3324	case SUPPAGINGMODE_AMD64_GLOBAL:
3325	case SUPPAGINGMODE_AMD64_NX:
3326	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3327	enmShadowMode = PGMMODE_PAE;
3328	break;
3329
3330	default:
3331	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3332	}
3333	break;
3334
3335	case PGMMODE_PAE:
3336	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3337	switch (enmHostMode)
3338	{
3339	case SUPPAGINGMODE_32_BIT:
3340	case SUPPAGINGMODE_32_BIT_GLOBAL:
3341	enmShadowMode = PGMMODE_PAE;
3342	break;
3343
3344	case SUPPAGINGMODE_PAE:
3345	case SUPPAGINGMODE_PAE_NX:
3346	case SUPPAGINGMODE_PAE_GLOBAL:
3347	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3348	enmShadowMode = PGMMODE_PAE;
3349	break;
3350
3351	case SUPPAGINGMODE_AMD64:
3352	case SUPPAGINGMODE_AMD64_GLOBAL:
3353	case SUPPAGINGMODE_AMD64_NX:
3354	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3355	enmShadowMode = PGMMODE_PAE;
3356	break;
3357
3358	default:
3359	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3360	}
3361	break;
3362
3363	case PGMMODE_AMD64:
3364	case PGMMODE_AMD64_NX:
3365	switch (enmHostMode)
3366	{
3367	case SUPPAGINGMODE_32_BIT:
3368	case SUPPAGINGMODE_32_BIT_GLOBAL:
3369	enmShadowMode = PGMMODE_AMD64;
3370	break;
3371
3372	case SUPPAGINGMODE_PAE:
3373	case SUPPAGINGMODE_PAE_NX:
3374	case SUPPAGINGMODE_PAE_GLOBAL:
3375	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3376	enmShadowMode = PGMMODE_AMD64;
3377	break;
3378
3379	case SUPPAGINGMODE_AMD64:
3380	case SUPPAGINGMODE_AMD64_GLOBAL:
3381	case SUPPAGINGMODE_AMD64_NX:
3382	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3383	enmShadowMode = PGMMODE_AMD64;
3384	break;
3385
3386	default:
3387	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3388	}
3389	break;
3390
3391	default:
3392	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3393	}
3394
3395	/*
3396	* Override the shadow mode when NEM, IEM or nested paging is active.
3397	*/
3398	if (!VM_IS_HM_ENABLED(pVM))
3399	{
3400	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
3401	pVM->pgm.s.fNestedPaging = true;
3402	enmShadowMode = PGMMODE_NONE;
3403	}
3404	else
3405	{
3406	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3407	pVM->pgm.s.fNestedPaging = fNestedPaging;
3408	if (fNestedPaging)
3409	{
3410	if (HMIsVmxActive(pVM))
3411	enmShadowMode = PGMMODE_EPT;
3412	else
3413	{
3414	/* The nested SVM paging depends on the host one. */
3415	Assert(HMIsSvmActive(pVM));
3416	if ( enmGuestMode == PGMMODE_AMD64
3417	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3418	enmShadowMode = PGMMODE_NESTED_AMD64;
3419	else
3420	switch (pVM->pgm.s.enmHostMode)
3421	{
3422	case SUPPAGINGMODE_32_BIT:
3423	case SUPPAGINGMODE_32_BIT_GLOBAL:
3424	enmShadowMode = PGMMODE_NESTED_32BIT;
3425	break;
3426
3427	case SUPPAGINGMODE_PAE:
3428	case SUPPAGINGMODE_PAE_GLOBAL:
3429	case SUPPAGINGMODE_PAE_NX:
3430	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3431	enmShadowMode = PGMMODE_NESTED_PAE;
3432	break;
3433
3434	case SUPPAGINGMODE_AMD64:
3435	case SUPPAGINGMODE_AMD64_GLOBAL:
3436	case SUPPAGINGMODE_AMD64_NX:
3437	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3438	enmShadowMode = PGMMODE_NESTED_AMD64;
3439	break;
3440
3441	default:
3442	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3443	}
3444	}
3445	}
3446	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3447	else
3448	{
3449	/* Nested paging is a requirement for nested VT-x. */
3450	AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3451	}
3452	#endif
3453	}
3454
3455	return enmShadowMode;
3456	}
3457
3458	#endif /* VBOX_VMM_TARGET_X86 */
3459
3460	/**
3461	* Performs the actual mode change.
3462	* This is called by PGMChangeMode and pgmR3InitPaging().
3463	*
3464	* @returns VBox status code. May suspend or power off the VM on error, but this
3465	* will trigger using FFs and not informational status codes.
3466	*
3467	* @param pVM The cross context VM structure.
3468	* @param pVCpu The cross context virtual CPU structure.
3469	* @param enmGuestMode The new guest mode. This is assumed to be different from
3470	* the current mode.
3471	* @param fForce Whether to force a shadow paging mode change.
3472	*/
3473	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
3474	{
3475	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3476	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3477
3478	#ifdef VBOX_VMM_TARGET_X86
3479	/*
3480	* Calc the shadow mode and switcher.
3481	*/
3482	PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3483	bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode \|\| fForce;
3484
3485	/*
3486	* Exit old mode(s).
3487	*/
3488	/* shadow */
3489	if (fShadowModeChanged)
3490	{
3491	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3492	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3493	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3494	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3495	{
3496	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3497	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3498	}
3499	}
3500	else
3501	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3502
3503	/* guest */
3504	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3505	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3506	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3507	{
3508	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3509	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3510	}
3511	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3512	pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
3513	pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
3514	Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
3515
3516	/*
3517	* Change the paging mode data indexes.
3518	*/
3519	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3520	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3521	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3522	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3523	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3524	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3525	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3526	# ifdef IN_RING3
3527	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3528	# endif
3529
3530	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3531	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3532	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3533	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3534	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3535	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3536	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3537	# ifdef IN_RING3
3538	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3539	# endif
3540
3541	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3542	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3543	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3544	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3545	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3546	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3547	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3548	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3549	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3550	# ifdef VBOX_STRICT
3551	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3552	# endif
3553
3554	/*
3555	* Determine SLAT mode -before- entering the new shadow mode!
3556	*/
3557	pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
3558
3559	/*
3560	* Enter new shadow mode (if changed).
3561	*/
3562	if (fShadowModeChanged)
3563	{
3564	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3565	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
3566	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3567	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3568	}
3569
3570	/*
3571	* Always flag the necessary updates
3572	*/
3573	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3574
3575	/*
3576	* Enter the new guest and shadow+guest modes.
3577	*/
3578	/* Calc the new CR3 value. */
3579	RTGCPHYS GCPhysCR3;
3580	switch (enmGuestMode)
3581	{
3582	case PGMMODE_REAL:
3583	case PGMMODE_PROTECTED:
3584	GCPhysCR3 = NIL_RTGCPHYS;
3585	break;
3586
3587	case PGMMODE_32_BIT:
3588	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3589	break;
3590
3591	case PGMMODE_PAE_NX:
3592	case PGMMODE_PAE:
3593	if (!pVM->cpum.ro.GuestFeatures.fPae)
3594	# ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3595	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3596	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3597	# else
3598	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3599
3600	# endif
3601	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3602	break;
3603
3604	# ifdef VBOX_WITH_64_BITS_GUESTS
3605	case PGMMODE_AMD64_NX:
3606	case PGMMODE_AMD64:
3607	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3608	break;
3609	# endif
3610	default:
3611	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3612	}
3613
3614	# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3615	/*
3616	* If a nested-guest is using EPT paging:
3617	* - Update the second-level address translation (SLAT) mode.
3618	* - Indicate that the CR3 is nested-guest physical address.
3619	*/
3620	if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
3621	{
3622	if (PGMMODE_WITH_PAGING(enmGuestMode))
3623	{
3624	/*
3625	* Translate CR3 to its guest-physical address.
3626	* We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
3627	* switching modes to keep it consistent with how GCPhysCR3 is updated.
3628	*/
3629	PGMPTWALK Walk;
3630	PGMPTWALKGST GstWalk;
3631	int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid /, 0 / GCPtrNested */, &Walk,
3632	&GstWalk);
3633	if (RT_SUCCESS(rc))
3634	{ /* likely */ }
3635	else
3636	{
3637	/*
3638	* SLAT failed but we avoid reporting this to the caller because the caller
3639	* is not supposed to fail. The only time the caller needs to indicate a
3640	* failure to software is when PAE paging is used by the nested-guest, but
3641	* we handle the PAE case separately (e.g., see VMX transition in IEM).
3642	* In all other cases, the failure will be indicated when CR3 tries to be
3643	* translated on the next linear-address memory access.
3644	* See Intel spec. 27.2.1 "EPT Overview".
3645	*/
3646	Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
3647
3648	/* Trying to coax PGM to succeed for the time being... */
3649	Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
3650	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3651	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3652	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3653	return VINF_SUCCESS;
3654	}
3655	pVCpu->pgm.s.GCPhysNstGstCR3 = GCPhysCR3;
3656	GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
3657	}
3658	}
3659	else
3660	Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
3661	# endif
3662
3663	/*
3664	* Enter the new guest mode.
3665	*/
3666	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3667	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3668	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3669
3670	/* Set the new guest CR3 (and nested-guest CR3). */
3671	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3672
3673	/* status codes. */
3674	AssertRC(rc);
3675	AssertRC(rc2);
3676	if (RT_SUCCESS(rc))
3677	{
3678	rc = rc2;
3679	if (RT_SUCCESS(rc)) /* no informational status codes. */
3680	rc = VINF_SUCCESS;
3681	}
3682
3683	/*
3684	* Notify HM.
3685	*/
3686	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3687	return rc;
3688
3689	#elif defined(VBOX_VMM_TARGET_ARMV8)
3690	//AssertReleaseFailed(); /** @todo Called by the PGM saved state code. */
3691	RT_NOREF(pVM, pVCpu, enmGuestMode, fForce);
3692	return VINF_SUCCESS;
3693
3694	#else
3695	# error "port me"
3696	#endif
3697	}
3698
3699
3700	#ifdef VBOX_VMM_TARGET_X86
3701	/**
3702	* Called by CPUM or REM when CR0.WP changes to 1.
3703	*
3704	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3705	* @thread EMT
3706	*/
3707	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3708	{
3709	/*
3710	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3711	*
3712	* Use the counter to judge whether there might be pool pages with active
3713	* hacks in them. If there are, we will be running the risk of messing up
3714	* the guest by allowing it to write to read-only pages. Thus, we have to
3715	* clear the page pool ASAP if there is the slightest chance.
3716	*/
3717	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3718	{
3719	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3720
3721	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3722	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3723	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3724	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3725	}
3726	}
3727	#endif /* VBOX_VMM_TARGET_X86 */
3728
3729
3730	/**
3731	* Gets the current guest paging mode.
3732	*
3733	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3734	*
3735	* @returns The current paging mode.
3736	* @param pVCpu The cross context virtual CPU structure.
3737	*/
3738	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3739	{
3740	return pVCpu->pgm.s.enmGuestMode;
3741	}
3742
3743
3744	/**
3745	* Gets the current shadow paging mode.
3746	*
3747	* @returns The current paging mode.
3748	* @param pVCpu The cross context virtual CPU structure.
3749	*/
3750	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3751	{
3752	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3753	return pVCpu->pgm.s.enmShadowMode;
3754	#else
3755	RT_NOREF(pVCpu);
3756	return PGMMODE_NONE;
3757	#endif
3758	}
3759
3760
3761	#ifdef VBOX_VMM_TARGET_X86
3762	/**
3763	* Gets the current host paging mode.
3764	*
3765	* @returns The current paging mode.
3766	* @param pVM The cross context VM structure.
3767	*/
3768	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3769	{
3770	switch (pVM->pgm.s.enmHostMode)
3771	{
3772	case SUPPAGINGMODE_32_BIT:
3773	case SUPPAGINGMODE_32_BIT_GLOBAL:
3774	return PGMMODE_32_BIT;
3775
3776	case SUPPAGINGMODE_PAE:
3777	case SUPPAGINGMODE_PAE_GLOBAL:
3778	return PGMMODE_PAE;
3779
3780	case SUPPAGINGMODE_PAE_NX:
3781	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3782	return PGMMODE_PAE_NX;
3783
3784	case SUPPAGINGMODE_AMD64:
3785	case SUPPAGINGMODE_AMD64_GLOBAL:
3786	return PGMMODE_AMD64;
3787
3788	case SUPPAGINGMODE_AMD64_NX:
3789	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3790	return PGMMODE_AMD64_NX;
3791
3792	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3793	}
3794
3795	return PGMMODE_INVALID;
3796	}
3797	#endif /* VBOX_VMM_TARGET_X86 */
3798
3799
3800	/**
3801	* Get mode name.
3802	*
3803	* @returns read-only name string.
3804	* @param enmMode The mode which name is desired.
3805	*/
3806	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3807	{
3808	switch (enmMode)
3809	{
3810	case PGMMODE_REAL: return "Real";
3811	case PGMMODE_PROTECTED: return "Protected";
3812	case PGMMODE_32_BIT: return "32-bit";
3813	case PGMMODE_PAE: return "PAE";
3814	case PGMMODE_PAE_NX: return "PAE+NX";
3815	case PGMMODE_AMD64: return "AMD64";
3816	case PGMMODE_AMD64_NX: return "AMD64+NX";
3817	case PGMMODE_NESTED_32BIT: return "Nested-32";
3818	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3819	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3820	case PGMMODE_EPT: return "EPT";
3821	case PGMMODE_NONE: return "None";
3822	case PGMMODE_VMSA_V8_32: return "VMSAv8-32";
3823	case PGMMODE_VMSA_V8_64: return "VMSAv8-64";
3824	default: return "unknown mode value";
3825	}
3826	}
3827
3828
3829	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
3830	/**
3831	* Gets the SLAT mode name.
3832	*
3833	* @returns The read-only SLAT mode descriptive string.
3834	* @param enmSlatMode The SLAT mode value.
3835	*/
3836	VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
3837	{
3838	switch (enmSlatMode)
3839	{
3840	case PGMSLAT_DIRECT: return "Direct";
3841	case PGMSLAT_EPT: return "EPT";
3842	case PGMSLAT_32BIT: return "32-bit";
3843	case PGMSLAT_PAE: return "PAE";
3844	case PGMSLAT_AMD64: return "AMD64";
3845	default: return "Unknown";
3846	}
3847	}
3848	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
3849
3850
3851	#ifdef VBOX_VMM_TARGET_X86
3852	/**
3853	* Notification from CPUM that the EFER.NXE bit has changed.
3854	*
3855	* @param pVCpu The cross context virtual CPU structure of the CPU for
3856	* which EFER changed.
3857	* @param fNxe The new NXE state.
3858	*/
3859	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3860	{
3861	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3862	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3863
3864	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3865	if (fNxe)
3866	{
3867	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3868	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3869	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3870	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3871	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3872	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3873	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3874	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3875	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3876	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3877	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3878
3879	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3880	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3881	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3882	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3883	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3884	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3885	}
3886	else
3887	{
3888	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3889	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3890	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3891	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3892	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3893	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3894	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3895	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3896	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3897	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3898	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3899
3900	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3901	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3902	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3903	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3904	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3905	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3906	}
3907	}
3908	#endif /* VBOX_VMM_TARGET_X86 */
3909
3910
3911	/**
3912	* Check if any pgm pool pages are marked dirty (not monitored)
3913	*
3914	* @returns bool locked/not locked
3915	* @param pVM The cross context VM structure.
3916	*/
3917	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3918	{
3919	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3920	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3921	#else
3922	RT_NOREF(pVM);
3923	return false;
3924	#endif
3925	}
3926
3927
3928	/**
3929	* Enable or disable large page usage
3930	*
3931	* @returns VBox status code.
3932	* @param pVM The cross context VM structure.
3933	* @param fUseLargePages Use/not use large pages
3934	*/
3935	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3936	{
3937	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3938
3939	pVM->pgm.s.fUseLargePages = fUseLargePages;
3940	return VINF_SUCCESS;
3941	}
3942
3943
3944	/**
3945	* Check if this VCPU currently owns the PGM lock.
3946	*
3947	* @returns bool owner/not owner
3948	* @param pVM The cross context VM structure.
3949	*/
3950	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3951	{
3952	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3953	}
3954
3955
3956	/**
3957	* Acquire the PGM lock.
3958	*
3959	* @returns VBox status code
3960	* @param pVM The cross context VM structure.
3961	* @param fVoid Set if the caller cannot handle failure returns.
3962	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3963	*/
3964	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3965	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3966	#else
3967	int pgmLock(PVMCC pVM, bool fVoid)
3968	#endif
3969	{
3970	#if defined(VBOX_STRICT)
3971	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3972	#else
3973	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3974	#endif
3975	if (RT_SUCCESS(rc))
3976	return rc;
3977	if (fVoid)
3978	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3979	else
3980	AssertRC(rc);
3981	return rc;
3982	}
3983
3984
3985	/**
3986	* Release the PGM lock.
3987	*
3988	* @param pVM The cross context VM structure.
3989	*/
3990	void pgmUnlock(PVMCC pVM)
3991	{
3992	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3993	pVM->pgm.s.cDeprecatedPageLocks = 0;
3994	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3995	if (rc == VINF_SEM_NESTED)
3996	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3997	}
3998
3999
4000	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4001
4002	/** Format handler for PGMPAGE.
4003	* @copydoc FNRTSTRFORMATTYPE */
4004	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
4005	const char pszType, void const pvValue,
4006	int cchWidth, int cchPrecision, unsigned fFlags,
4007	void *pvUser)
4008	{
4009	size_t cch;
4010	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
4011	if (RT_VALID_PTR(pPage))
4012	{
4013	char szTmp[64+80];
4014
4015	cch = 0;
4016
4017	/* The single char state stuff. */
4018	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
4019	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
4020
4021	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
4022	if (IS_PART_INCLUDED(5))
4023	{
4024	static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
4025	szTmp[cch++] = s_achHandlerStates[ PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
4026	\| ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
4027	}
4028
4029	/* The type. */
4030	if (IS_PART_INCLUDED(4))
4031	{
4032	szTmp[cch++] = ':';
4033	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
4034	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
4035	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
4036	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
4037	}
4038
4039	/* The numbers. */
4040	if (IS_PART_INCLUDED(3))
4041	{
4042	szTmp[cch++] = ':';
4043	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
4044	}
4045
4046	if (IS_PART_INCLUDED(2))
4047	{
4048	szTmp[cch++] = ':';
4049	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
4050	}
4051
4052	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
4053	if (IS_PART_INCLUDED(6))
4054	{
4055	szTmp[cch++] = ':';
4056	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
4057	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
4058	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
4059	}
4060	# endif
4061	# undef IS_PART_INCLUDED
4062
4063	cch = pfnOutput(pvArgOutput, szTmp, cch);
4064	#if 0
4065	size_t cch2 = 0;
4066	szTmp[cch2++] = '(';
4067	cch2 += RTStrFormatNumber(&szTmp[cch2], (uintptr_t)pPage, 16, 18, 0, RTSTR_F_SPECIAL \| RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
4068	szTmp[cch2++] = ')';
4069	szTmp[cch2] = '\0';
4070	cch += pfnOutput(pvArgOutput, szTmp, cch2);
4071	#endif
4072	}
4073	else
4074	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
4075	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
4076	return cch;
4077	}
4078
4079
4080	/** Format handler for PGMRAMRANGE.
4081	* @copydoc FNRTSTRFORMATTYPE */
4082	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
4083	const char pszType, void const pvValue,
4084	int cchWidth, int cchPrecision, unsigned fFlags,
4085	void *pvUser)
4086	{
4087	size_t cch;
4088	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
4089	if (RT_VALID_PTR(pRam))
4090	{
4091	char szTmp[80];
4092	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
4093	cch = pfnOutput(pvArgOutput, szTmp, cch);
4094	}
4095	else
4096	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
4097	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
4098	return cch;
4099	}
4100
4101	/** Format type andlers to be registered/deregistered. */
4102	static const struct
4103	{
4104	char szType[24];
4105	PFNRTSTRFORMATTYPE pfnHandler;
4106	} g_aPgmFormatTypes[] =
4107	{
4108	{ "pgmpage", pgmFormatTypeHandlerPage },
4109	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
4110	};
4111
4112	#endif /* !IN_R0 \|\| LOG_ENABLED */
4113
4114	/**
4115	* Registers the global string format types.
4116	*
4117	* This should be called at module load time or in some other manner that ensure
4118	* that it's called exactly one time.
4119	*
4120	* @returns IPRT status code on RTStrFormatTypeRegister failure.
4121	*/
4122	VMMDECL(int) PGMRegisterStringFormatTypes(void)
4123	{
4124	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4125	int rc = VINF_SUCCESS;
4126	unsigned i;
4127	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4128	{
4129	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4130	# ifdef IN_RING0
4131	if (rc == VERR_ALREADY_EXISTS)
4132	{
4133	/* in case of cleanup failure in ring-0 */
4134	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4135	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
4136	}
4137	# endif
4138	}
4139	if (RT_FAILURE(rc))
4140	while (i-- > 0)
4141	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4142
4143	return rc;
4144	#else
4145	return VINF_SUCCESS;
4146	#endif
4147	}
4148
4149
4150	/**
4151	* Deregisters the global string format types.
4152	*
4153	* This should be called at module unload time or in some other manner that
4154	* ensure that it's called exactly one time.
4155	*/
4156	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
4157	{
4158	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
4159	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
4160	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
4161	#endif
4162	}
4163
4164	#ifdef VBOX_VMM_TARGET_X86
4165
4166	# ifdef VBOX_STRICT
4167	/**
4168	* Asserts that everything related to the guest CR3 is correctly shadowed.
4169	*
4170	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
4171	* and assert the correctness of the guest CR3 mapping before asserting that the
4172	* shadow page tables is in sync with the guest page tables.
4173	*
4174	* @returns Number of conflicts.
4175	* @param pVM The cross context VM structure.
4176	* @param pVCpu The cross context virtual CPU structure.
4177	* @param cr3 The current guest CR3 register value.
4178	* @param cr4 The current guest CR4 register value.
4179	*/
4180	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
4181	{
4182	AssertReturn(pVM->enmTarget == VMTARGET_X86, 0);
4183	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4184
4185	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
4186	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
4187	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
4188
4189	PGM_LOCK_VOID(pVM);
4190	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
4191	PGM_UNLOCK(pVM);
4192
4193	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
4194	return cErrors;
4195	}
4196	# endif /* VBOX_STRICT */
4197
4198
4199	/**
4200	* Updates PGM's copy of the guest's EPT pointer.
4201	*
4202	* @param pVCpu The cross context virtual CPU structure.
4203	* @param uEptPtr The EPT pointer.
4204	*
4205	* @remarks This can be called as part of VM-entry so we might be in the midst of
4206	* switching to VMX non-root mode.
4207	*/
4208	VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
4209	{
4210	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4211	PGM_LOCK_VOID(pVM);
4212	pVCpu->pgm.s.uEptPtr = uEptPtr;
4213	pVCpu->pgm.s.pGstEptPml4R3 = 0;
4214	pVCpu->pgm.s.pGstEptPml4R0 = 0;
4215	PGM_UNLOCK(pVM);
4216	}
4217
4218	#endif /* VBOX_VMM_TARGET_X86 */
4219	#ifdef PGM_WITH_PAGE_ZEROING_DETECTION
4220	# ifndef VBOX_VMM_TARGET_X86
4221	# error "misconfig: PGM_WITH_PAGE_ZEROING_DETECTION not implemented for ARM guests"
4222	# endif
4223
4224	/**
4225	* Helper for checking whether XMM0 is zero, possibly retriving external state.
4226	*/
4227	static bool pgmHandlePageZeroingIsXmm0Zero(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4228	{
4229	if (pCtx->fExtrn & CPUMCTX_EXTRN_SSE_AVX)
4230	{
4231	int rc = CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_SSE_AVX);
4232	AssertRCReturn(rc, false);
4233	}
4234	return pCtx->XState.x87.aXMM[0].au64[0] == 0
4235	&& pCtx->XState.x87.aXMM[0].au64[1] == 0
4236	&& pCtx->XState.x87.aXMM[0].au64[2] == 0
4237	&& pCtx->XState.x87.aXMM[0].au64[3] == 0;
4238	}
4239
4240
4241	/**
4242	* Helper for comparing opcode bytes.
4243	*/
4244	static bool pgmHandlePageZeroingMatchOpcodes(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t const *pbOpcodes, uint32_t cbOpcodes)
4245	{
4246	uint8_t abTmp[64];
4247	AssertMsgReturn(cbOpcodes <= sizeof(abTmp), ("cbOpcodes=%#x\n", cbOpcodes), false);
4248	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abTmp, pCtx->rip + pCtx->cs.u64Base, cbOpcodes);
4249	if (RT_SUCCESS(rc))
4250	return memcmp(abTmp, pbOpcodes, cbOpcodes) == 0;
4251	return false;
4252	}
4253
4254
4255	/**
4256	* Called on faults on ZERO pages to check if the guest is trying to zero it.
4257	*
4258	* Since it's a waste of time to zero a ZERO page and it will cause an
4259	* unnecessary page allocation, we'd like to detect and avoid this.
4260	* If any known page zeroing code is detected, this function will update the CPU
4261	* state to pretend the page was zeroed by the code.
4262	*
4263	* @returns true if page zeroing code was detected and CPU state updated to skip
4264	* the code.
4265	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4266	* @param pCtx The guest register context.
4267	*/
4268	static bool pgmHandlePageZeroingCode(PVMCPUCC pVCpu, PCPUMCTX pCtx)
4269	{
4270	CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
4271
4272	/*
4273	* Sort by mode first.
4274	*/
4275	if (CPUMIsGuestInLongModeEx(pCtx))
4276	{
4277	if (CPUMIsGuestIn64BitCodeEx(pCtx))
4278	{
4279	/*
4280	* 64-bit code.
4281	*/
4282	Log9(("pgmHandlePageZeroingCode: not page zeroing - 64-bit\n"));
4283	}
4284	else if (pCtx->cs.Attr.n.u1DefBig)
4285	Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit lm\n"));
4286	else
4287	Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit lm\n"));
4288	}
4289	else if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
4290	{
4291	if (pCtx->cs.Attr.n.u1DefBig)
4292	{
4293	/*
4294	* 32-bit paged protected mode code.
4295	*/
4296	CPUMCTX_ASSERT_NOT_EXTRN(pCtx, CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RBX
4297	\| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP \| CPUMCTX_EXTRN_RSI \| CPUMCTX_EXTRN_RDI
4298	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
4299
4300	/* 1. Generic 'rep stosd' detection. */
4301	static uint8_t const s_abRepStosD[] = { 0xf3, 0xab };
4302	if ( pCtx->eax == 0
4303	&& pCtx->ecx == X86_PAGE_SIZE / 4
4304	&& !(pCtx->edi & X86_PAGE_OFFSET_MASK)
4305	&& pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abRepStosD, sizeof(s_abRepStosD)))
4306	{
4307	pCtx->ecx = 0;
4308	pCtx->edi += X86_PAGE_SIZE;
4309	Log9(("pgmHandlePageZeroingCode: REP STOSD: eip=%RX32 -> %RX32\n", pCtx->eip, pCtx->eip + sizeof(s_abRepStosD)));
4310	pCtx->eip += sizeof(s_abRepStosD);
4311	return true;
4312	}
4313
4314	/* 2. Windows 2000 sp4 KiXMMIZeroPageNoSave loop code: */
4315	static uint8_t const s_abW2kSp4XmmZero[] =
4316	{
4317	0x0f, 0x2b, 0x01,
4318	0x0f, 0x2b, 0x41, 0x10,
4319	0x0f, 0x2b, 0x41, 0x20,
4320	0x0f, 0x2b, 0x41, 0x30,
4321	0x83, 0xc1, 0x40,
4322	0x48,
4323	0x75, 0xeb,
4324	};
4325	if ( pCtx->eax == 64
4326	&& !(pCtx->ecx & X86_PAGE_OFFSET_MASK)
4327	&& pgmHandlePageZeroingMatchOpcodes(pVCpu, pCtx, s_abW2kSp4XmmZero, sizeof(s_abW2kSp4XmmZero))
4328	&& pgmHandlePageZeroingIsXmm0Zero(pVCpu, pCtx))
4329	{
4330	pCtx->eax = 1;
4331	pCtx->ecx += X86_PAGE_SIZE;
4332	Log9(("pgmHandlePageZeroingCode: w2k sp4 xmm: eip=%RX32 -> %RX32\n",
4333	pCtx->eip, pCtx->eip + sizeof(s_abW2kSp4XmmZero) - 3));
4334	pCtx->eip += sizeof(s_abW2kSp4XmmZero) - 3;
4335	return true;
4336	}
4337	Log9(("pgmHandlePageZeroingCode: not page zeroing - 32-bit\n"));
4338	}
4339	else if (!pCtx->eflags.Bits.u1VM)
4340	Log9(("pgmHandlePageZeroingCode: not page zeroing - 16-bit\n"));
4341	else
4342	Log9(("pgmHandlePageZeroingCode: not page zeroing - v86\n"));
4343	}
4344	return false;
4345	}
4346
4347	#endif /* PGM_WITH_PAGE_ZEROING_DETECTION */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 108134

Download in other formats: